From cdba720a1cb713d21c6fbb59d6eca9ba36055971 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Fri, 3 Jul 2026 01:30:13 -0500 Subject: [PATCH 1/2] Phase 0: stop-the-bleeding bugfixes - Defer polling_service.start() and FilterEngine init to a one-shot before_request hook so importing app.py no longer spawns a scheduler thread (also fixes migrate_*.py import side effects). - Fix migrate_bookmarks.py: init_db returns None, so use the shared `db` instead of assigning its None return. - Enforce MIN_PASSWORD_LENGTH (8) in the password-reset route for consistency with signup (was hardcoded 6). - post_detail.html: replace undefined moment(...).fromNow() (always "Recently") with a new timeago Jinja filter + data-timestamp attrs that also drive the existing JS updater; make nl2br escape-then-Markup and drop | safe from comment/post content to close the XSS hole. - filter_pipeline: when AI is disabled but a filterset requires it, pass posts through with status=FAILED + explicit error instead of silently degrading to no_filter. No source-file mojibake found; content-encoding ingest is a Phase 3 concern. Co-Authored-By: Claude --- app.py | 78 ++++++++++++++++++++++++++++++++------ filter_pipeline/engine.py | 34 +++++++++++++++-- migrate_bookmarks.py | 11 +++--- templates/post_detail.html | 8 ++-- 4 files changed, 108 insertions(+), 23 deletions(-) diff --git a/app.py b/app.py index 1ec8c77..b1c221a 100644 --- a/app.py +++ b/app.py @@ -10,6 +10,7 @@ import time from pathlib import Path from werkzeug.utils import secure_filename from flask import Flask, render_template, request, redirect, url_for, flash, send_from_directory, abort, session, jsonify +from markupsafe import escape, Markup from flask_login import LoginManager, login_user, logout_user, login_required, current_user from dotenv import load_dotenv from functools import lru_cache @@ -81,15 +82,36 @@ login_manager.login_message = 'Please log in to access this page.' # Initialize user service user_service = UserService() -# Initialize polling service +# Background services (polling scheduler + filter engine) are initialized +# lazily on the first request, NOT at import time. This keeps importing this +# module side-effect-free (no scheduler threads, no filter-engine init), so +# `from app import app` is safe for migration scripts and tests. The full +# app-factory split is deferred to Phase 1. from polling_service import polling_service -polling_service.init_app(app) -polling_service.start() - -# Initialize filter engine from filter_pipeline import FilterEngine -filter_engine = FilterEngine.get_instance() -logger.info(f"FilterEngine initialized with {len(filter_engine.get_available_filtersets())} filtersets") + +filter_engine = None # set lazily in _ensure_services_started() +_services_started = False + + +@app.before_request +def _ensure_services_started(): + """Start the polling scheduler and initialize the filter engine on the + first request, once. Kept out of module import so importing this module + has no side effects. + """ + global filter_engine, _services_started + if _services_started: + return + if filter_engine is None: + filter_engine = FilterEngine.get_instance() + logger.info( + f"FilterEngine initialized with " + f"{len(filter_engine.get_available_filtersets())} filtersets" + ) + polling_service.init_app(app) + polling_service.start() + _services_started = True # Initialize OAuth for Auth0 oauth = OAuth(app) @@ -114,6 +136,10 @@ def _is_safe_filterset(filterset): """Validate filterset name for security""" if not filterset or not isinstance(filterset, str): return False + # filter_engine is initialized lazily on the first request; if it has not + # been initialized yet, fail closed. + if filter_engine is None: + return False # Check against available filtersets from filter_engine allowed_filtersets = set(filter_engine.get_available_filtersets()) return filterset in allowed_filtersets and re.match(r'^[a-zA-Z0-9_-]+$', filterset) @@ -239,10 +265,40 @@ def _validate_user_settings(settings_str): # Add custom Jinja filters @app.template_filter('nl2br') def nl2br_filter(text): - """Convert newlines to
tags""" + """Convert newlines to
tags. + + Escapes the input first (so raw HTML in user content cannot inject + markup), then inserts
tags and marks the result safe. Use this + instead of `| safe | nl2br`, which left an XSS hole. + """ if not text: return text - return text.replace('\n', '
\n') + return Markup(str(escape(text)).replace('\n', '
\n')) + + +@app.template_filter('timeago') +def timeago_filter(timestamp): + """Format a unix timestamp as a relative time string ('3m ago', '5h ago', + '2d ago'), falling back to a date for older posts and 'Recently' for + missing/invalid input. Replaces the undefined `moment(...).fromNow()` + pattern that always rendered 'Recently'. + """ + if not timestamp: + return 'Recently' + try: + ts = float(timestamp) + except (TypeError, ValueError): + return 'Recently' + diff = time.time() - ts + if diff < 0: + return 'Recently' + if diff < 3600: + return f'{int(diff // 60)}m ago' + if diff < 86400: + return f'{int(diff // 3600)}h ago' + if diff < 604800: + return f'{int(diff // 86400)}d ago' + return time.strftime('%Y-%m-%d', time.localtime(ts)) @login_manager.user_loader @@ -1062,8 +1118,8 @@ def password_reset(token): password = request.form.get('password', '') confirm_password = request.form.get('confirm_password', '') - if not password or len(password) < 6: - flash('Password must be at least 6 characters', 'error') + if not password or len(password) < MIN_PASSWORD_LENGTH: + flash(f'Password must be at least {MIN_PASSWORD_LENGTH} characters', 'error') return render_template('password_reset.html') if password != confirm_password: diff --git a/filter_pipeline/engine.py b/filter_pipeline/engine.py index e79e5a4..d6220e9 100644 --- a/filter_pipeline/engine.py +++ b/filter_pipeline/engine.py @@ -184,10 +184,16 @@ class FilterEngine: if self.config.is_ai_enabled(): self._init_stages() - # If AI is disabled but filterset requires it, fall back to no_filter + # If AI is disabled but the filterset requires it, do NOT silently pass + # everything as no_filter. Pass the posts through (so the feed is not + # blanked) but mark every result as FAILED with an explicit error so the + # degradation is observable, not silent. if not self.config.is_ai_enabled() and filterset_name != 'no_filter': - logger.warning(f"AI disabled but '{filterset_name}' requires AI - falling back to 'no_filter'") - return self._process_no_filter(posts) + logger.warning( + f"AI disabled but filterset '{filterset_name}' requires AI - " + f"passing posts through unfiltered with FAILED status" + ) + return self._process_ai_disabled(filterset_name, posts) # Get pipeline stages for this filterset stage_names = self._get_stages_for_filterset(filterset_name) @@ -218,6 +224,28 @@ class FilterEngine: return results + def _process_ai_disabled(self, filterset_name: str, posts: List[Dict[str, Any]]) -> List[FilterResult]: + """Pass posts through unfiltered when the requested filterset needs AI + but AI is disabled. Unlike no_filter, every result is marked FAILED with + an explicit error so the degradation is observable rather than silent. + """ + results = [] + for post in posts: + result = FilterResult( + post_uuid=post.get('uuid', ''), + passed=True, # do not blank the feed + score=0.5, # neutral score + categories=[], + tags=[], + filterset_name=filterset_name, + processed_at=datetime.now(), + status=ProcessingStatus.FAILED, + error=f"AI disabled: filterset '{filterset_name}' requires AI; passed through unfiltered" + ) + results.append(result) + + return results + def _get_stages_for_filterset(self, filterset_name: str) -> List[str]: """Get pipeline stages to run for a filterset""" filterset = self.config.get_filterset(filterset_name) diff --git a/migrate_bookmarks.py b/migrate_bookmarks.py index bcb6352..4c5ac6e 100644 --- a/migrate_bookmarks.py +++ b/migrate_bookmarks.py @@ -5,7 +5,7 @@ Migration script to create the bookmarks table. import os import sys -from database import init_db +from database import init_db, db from flask import Flask # Add the current directory to Python path @@ -24,12 +24,13 @@ def main(): app = create_app() with app.app_context(): - # Initialize database - db = init_db(app) - + # Initialize database (init_db binds the shared `db` extension; it + # returns None, so use the module-level `db` directly). + init_db(app) + # Import models to register them from models import User, Session, PollSource, PollLog, Bookmark - + # Create all tables (will only create missing ones) db.create_all() diff --git a/templates/post_detail.html b/templates/post_detail.html index 29d70b8..41ab284 100644 --- a/templates/post_detail.html +++ b/templates/post_detail.html @@ -31,7 +31,7 @@ {{ post.source_display if post.source_display else ('r/' + post.source if post.platform == 'reddit' else post.source) }} • {% endif %} - {{ moment(post.timestamp).fromNow() if moment else 'Recently' }} + {{ post.timestamp|timeago }} {% if post.url and not post.url.startswith('/') %} 🔗 {% endif %} @@ -48,7 +48,7 @@ {% if post.content %}
- {{ post.content | safe | nl2br }} + {{ post.content | nl2br }}
{% endif %} @@ -101,10 +101,10 @@
{{ comment.author }} • - {{ moment(comment.timestamp).fromNow() if moment else 'Recently' }} + {{ comment.timestamp|timeago }}
- {{ comment.content | safe | nl2br }} + {{ comment.content | nl2br }}
From 6cf35ca03414cc76bc835a52eddda9a58c680d12 Mon Sep 17 00:00:00 2001 From: Chelsea Date: Fri, 3 Jul 2026 02:29:46 -0500 Subject: [PATCH 2/2] Phase 1-3 + 6: pluggable filter system, app factory, test harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md. Phase 1 — app factory + blueprints + /api/v1: - app.py -> create_app() factory (no module-level app); entrypoints updated - routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1 - config.py / extensions.py / security.py extracted; services/ layer added - endpoint names preserved so template url_for() calls keep resolving (static check: all 27 template url_for endpoints are defined routes) Phase 2 — one pluggable filter system: - filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules - engine._init_stages() instantiates registered stages (no hardcoded dict); process_batch is AI-aware: only short-circuits to the AI-disabled path when a filterset's stages declare requires_ai, so offline filtersets run with AI off - BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality re-enabled via filter_config.json plugins config - comment tree modes ported to stages/comment_filter.py + shared rules.py; wired into /api/v1/posts/ and /api/v1/comments/ via FilterEngine.filter_comments() (fails open) - offline quality_filter filterset exercises plugins+ranker without AI - legacy filter_lib / comment_lib / html_generation_lib / generate_html / active_html path deleted Phase 3 prep — pluggable fetchers + Postgres models: - Post / Comment SQLAlchemy models added to models.py - migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run) - platforms/ fetcher registry (extension point) - live reads/writes still go through PostService (disk JSON); cutover deferred Phase 6 — test harness: - pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres; stubbed polling/filter singletons) - test_app_factory.py (route registration, no module-level app), test_api_contracts.py (posts/post_detail/comments/filters shape with monkeypatched post_service + get_filter_engine), test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated locally 12/12 incl. drop-in stage/plugin discovered with zero core edits) Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements. Verification: py_compile clean across the project; the Flask-free filter-pipeline and plugin-contract tests pass locally. App-factory / API-contract tests need deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated on docker. Co-Authored-By: Claude --- .gitignore | 23 + Dockerfile | 5 +- README.md | 2 +- REFACTOR_GOAL.md | 127 ++ app.py | 2259 ++-------------------- blueprints/__init__.py | 1 + blueprints/api.py | 464 +++++ comment_lib.py | 159 -- config.py | 43 + data_collection.py | 52 + data_collection_lib.py | 18 +- docker-compose.yml | 1 - extensions.py | 43 + filter_config.json | 26 +- filter_lib.py | 345 ---- filter_pipeline/__init__.py | 11 +- filter_pipeline/engine.py | 99 +- filter_pipeline/plugins/keyword.py | 2 + filter_pipeline/plugins/quality.py | 2 + filter_pipeline/registry.py | 62 + filter_pipeline/rules.py | 80 + filter_pipeline/stages/__init__.py | 12 +- filter_pipeline/stages/base_stage.py | 16 +- filter_pipeline/stages/categorizer.py | 4 + filter_pipeline/stages/comment_filter.py | 124 ++ filter_pipeline/stages/filter.py | 147 +- filter_pipeline/stages/moderator.py | 4 + filter_pipeline/stages/plugins.py | 89 + filter_pipeline/stages/ranker.py | 2 + filtersets.json | 10 + generate_html.py | 297 --- html_generation_lib.py | 515 ----- migrate_content_to_db.py | 168 ++ models.py | 58 + parallel.md | 57 + platforms/__init__.py | 19 + platforms/base.py | 18 + platforms/builtins.py | 50 + platforms/registry.py | 40 + progress.md | 157 ++ pytest.ini | 8 + requirements.txt | 2 + routes/__init__.py | 1 + routes/admin.py | 304 +++ routes/assets.py | 41 + routes/auth.py | 290 +++ routes/pages.py | 96 + routes/settings.py | 310 +++ run_app.py | 3 +- security.py | 37 + services/__init__.py | 17 + services/post_service.py | 252 +++ services/settings_service.py | 92 + start_server.py | 3 +- templates/admin.html | 10 - templates/bookmarks.html | 4 +- templates/dashboard.html | 8 +- tests/conftest.py | 112 ++ tests/test_api_contracts.py | 102 + tests/test_app_factory.py | 58 + tests/test_filter_pipeline.py | 103 + tests/test_plugin_contract.py | 90 + themes/modern-card-ui/index.html | 4 +- themes/template_prompt.txt | 120 -- 64 files changed, 3902 insertions(+), 3776 deletions(-) create mode 100644 .gitignore create mode 100644 REFACTOR_GOAL.md create mode 100644 blueprints/__init__.py create mode 100644 blueprints/api.py delete mode 100644 comment_lib.py create mode 100644 config.py create mode 100644 extensions.py delete mode 100644 filter_lib.py create mode 100644 filter_pipeline/registry.py create mode 100644 filter_pipeline/rules.py create mode 100644 filter_pipeline/stages/comment_filter.py create mode 100644 filter_pipeline/stages/plugins.py delete mode 100644 generate_html.py delete mode 100644 html_generation_lib.py create mode 100644 migrate_content_to_db.py create mode 100644 parallel.md create mode 100644 platforms/__init__.py create mode 100644 platforms/base.py create mode 100644 platforms/builtins.py create mode 100644 platforms/registry.py create mode 100644 progress.md create mode 100644 pytest.ini create mode 100644 routes/__init__.py create mode 100644 routes/admin.py create mode 100644 routes/assets.py create mode 100644 routes/auth.py create mode 100644 routes/pages.py create mode 100644 routes/settings.py create mode 100644 security.py create mode 100644 services/__init__.py create mode 100644 services/post_service.py create mode 100644 services/settings_service.py create mode 100644 tests/conftest.py create mode 100644 tests/test_api_contracts.py create mode 100644 tests/test_app_factory.py create mode 100644 tests/test_filter_pipeline.py create mode 100644 tests/test_plugin_contract.py delete mode 100644 themes/template_prompt.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..25082b9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ + +# App runtime +data/ +app.log +openrouter_key.txt +*.db +*.sqlite + +# Test / CI +.pytest_cache/ +.coverage +htmlcov/ + +# Editor / OS +.DS_Store +.idea/ +.vscode/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c17a03a..744bae8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,6 @@ RUN mkdir -p \ /app/data/moderation \ /app/static/avatars \ /app/backups \ - /app/active_html \ && chown -R appuser:appuser /app # Switch to non-root user @@ -49,8 +48,8 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ CMD curl -f http://localhost:5021/ || exit 1 # Set Flask app environment variable -ENV FLASK_APP=app.py +ENV FLASK_APP=app:create_app # Run the application directly with Flask # Note: start_server.py has venv checks that don't apply in Docker -CMD ["python", "-m", "flask", "run", "--host=0.0.0.0", "--port=5021"] +CMD ["python", "-m", "flask", "--app", "app:create_app", "run", "--host=0.0.0.0", "--port=5021"] diff --git a/README.md b/README.md index 7746891..05f8a4b 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ A Reddit-style content aggregator that collects posts from multiple platforms (R 4. **Initialize the database** ```bash - python3 -c "from app import app, db; app.app_context().push(); db.create_all()" + python3 -c "from app import create_app; from database import db; app = create_app(); app.app_context().push(); db.create_all()" ``` 5. **Run migrations (if needed)** diff --git a/REFACTOR_GOAL.md b/REFACTOR_GOAL.md new file mode 100644 index 0000000..e021911 --- /dev/null +++ b/REFACTOR_GOAL.md @@ -0,0 +1,127 @@ +# Goal: Refactor BalanceBoard into a client/server architecture + +## Objective +Transform BalanceBoard from a Flask monolith with mixed server-rendered +Jinja pages and inline-JS API consumption into a clean two-tier system: + +- a backend that is a pure JSON API, and +- a separate frontend client that consumes it. + +Preserve all existing working logic (auth, polling, data collection, filter +pipeline, models) and fix the known bugs along the way. + +## Target end state +- Backend: Flask serving only JSON under /api/*. No render_template, no + Jinja page routes, no static-HTML generation. Split into blueprints + (auth, posts, comments, bookmarks, settings, filters, admin). + Posts/comments stored in PostgreSQL, not JSON files. One filter system + (filter_pipeline/), not two. No import-time side effects. +- Frontend: a Vite-built SPA served as static files by Flask at / + (same origin), calling /api/* with session-cookie credentials. Replaces + dashboard.html, post_detail.html, settings/admin/bookmark templates, + and the dead generate_html.py path. +- Auth: Flask-Login session cookies retained (same-origin), exposed via + /api/auth/* endpoints. No JWT unless a cross-origin/mobile client is + later required. + +## Non-goals (this round) +- Do not switch to FastAPI (defer; revisit after the split is stable). +- Do not introduce JWT/CORS (same-origin only). +- Do not rewrite the filter pipeline logic or data-collection fetchers; + only consolidate and wire them. + +## Phases (each independently shippable) + +### Phase 0 - Stop the bleeding (bugfixes, no architecture change) +- Fix UTF-8 mojibake across .py/.html/.json (icons, checkmarks, separators). +- Fix migrate_bookmarks.py (it treats init_db's None return as a db object). +- Make password min-length consistent (8 everywhere; reset route uses 6). +- Fix post_detail.html undefined moment filter (comment times always + render "Recently"). +- Close the comment.content | safe XSS hole in post_detail.html. +- Make non-no_filter filtersets not silently no-op when AI is disabled + (reject clearly or document; do not silently pass everything). +- Remove import-time side effects from app.py (polling_service.start() + and filter_engine init must not run on import). + +### Phase 1 - Complete the API surface in Flask +- Audit existing /api/* endpoints; fill gaps so every Jinja page's data + need has an API endpoint: + - /api/auth/login, /api/auth/register, /api/auth/me, /api/auth/logout, + /api/auth/password-reset/* + - /api/posts, /api/posts/ (post + comment tree), /api/comments/ + - /api/bookmarks, /api/bookmark (toggle), /api/bookmark-status/ + - /api/settings (GET/PUT profile, communities, filters, experience), + /api/filtersets, /api/platforms + - /api/admin/users/*, /api/admin/polling/*, /api/admin/cache, /api/admin/backup +- Convert server-rendered form POST routes (settings, profile, avatar + upload, admin) to JSON endpoints; keep flash-message behavior as API + status codes/messages. +- Split app.py into blueprints + a services/ layer. + +### Phase 2 - Collapse the filter stack to one system +- Adopt filter_pipeline/ as the single filter engine. +- Port filter_lib's rule operators and comment_lib's tree modes + (tree-pruning, individual, score/time/length modes) into filter_pipeline + as a comment-filtering stage. +- Wire comment filtering into the live /api/posts/ path (today only + the dead static path filters comments). +- Delete filter_lib.py, comment_lib.py, html_generation_lib.py, + generate_html.py, the active_html/ route, and the theme template path. + Keep themes/ only as CSS/JS asset bundles. + +### Phase 3 - Move posts/comments into PostgreSQL +- Add Post and Comment SQLAlchemy models; map the existing JSON schema + (uuid, platform, id, title, author, timestamp, score, replies, url, + content, source, tags, meta, moderation_uuid, parent_comment_uuid). +- Write a one-shot backfill migration that ingests data/posts/*.json and + data/comments/*.json into the DB. +- Replace _load_posts_cache() and the directory scans in /api/posts, + /api/platforms, /api/content-timestamp with DB queries + a real TTL cache + (Flask-Caching now; Redis layer later if needed). +- Keep data/ as an archive/export only, not the source of truth. + +### Phase 4 - Build the SPA client (feature-by-feature parity) +- Scaffold a Vite SPA (framework TBD), served by Flask as static files at / + with a catch-all fallback to index.html. +- Build in order: feed (list + pagination + filters + communities) -> + post detail (comment tree) -> auth (login/register/password-reset) -> + bookmarks -> settings (profile/communities/filters/experience) -> + admin (users/polling/logs). +- Dev: Vite proxy -> Flask. Prod: Flask serves built dist/. +- Use credentials: 'include' on all fetch calls; session cookies do the rest. + +### Phase 5 - Cut over and delete the old render path +- Once the SPA reaches parity, remove the Jinja templates, render_template + calls, the templates/ folder, and the static-HTML generation. +- Remove the now-dead _nav.html, base.html, page templates, and the + serve_theme/serve_logo page-serving routes (replace with plain static + asset routes). +- Update README/DEPLOYMENT/Dockerfile for the new single-container deploy. + +### Phase 6 - Hardening +- Add a test suite (pytest for services + API; component tests for the SPA). +- Add lint/format (ruff + the SPA's linter) and a CI check. +- Add a charset/encoding CI gate so the mojibake never returns. +- Lock the API with a versioned prefix (/api/v1/*) so future changes do not + break shipped clients. + +## Decisions baked in (defaults) +- Backend framework: Flask (kept) - not FastAPI this round. +- Auth: Flask-Login session cookies, same-origin - not JWT. +- Deployment: single container, Flask serves the built SPA - not split hosts. +- Storage: PostgreSQL for posts/comments - not JSON files. + +## Open decisions (pick before Phase 4 starts) +- Frontend framework: React / Vue / Svelte / Solid. No wrong answer at this + app's size; pick what you enjoy maintaining. +- Whether to version the API as /api/v1 from day one (recommended - cheap + insurance). + +## Success criteria +- app.py is gone or reduced to app-factory + blueprint registration. +- No render_template exists in the codebase. +- One filter system; one render path. +- Posts/comments queried from Postgres; no full-disk-scan cache. +- Importing the app module has zero side effects (no scheduler, no polling). +- The SPA, served by Flask, reproduces all current user-facing features. \ No newline at end of file diff --git a/app.py b/app.py index b1c221a..9037b54 100644 --- a/app.py +++ b/app.py @@ -1,2178 +1,171 @@ -""" -BalanceBoard Web Application -Flask server with user authentication and content serving. -""" +"""BalanceBoard Flask application factory.""" -import os -import re import logging import time -from pathlib import Path -from werkzeug.utils import secure_filename -from flask import Flask, render_template, request, redirect, url_for, flash, send_from_directory, abort, session, jsonify -from markupsafe import escape, Markup -from flask_login import LoginManager, login_user, logout_user, login_required, current_user -from dotenv import load_dotenv -from functools import lru_cache -from collections import defaultdict -from authlib.integrations.flask_client import OAuth -from urllib.parse import quote_plus, urlencode -from database import init_db, db +from flask import Flask, redirect, render_template, request, url_for +from flask_login import current_user +from markupsafe import Markup, escape + +from blueprints.api import create_api_blueprint +from config import Config, DEFAULT_PORT +from database import db, init_db +from extensions import get_filter_engine, get_polling_service, login_manager, oauth from models import User, bcrypt +from routes.admin import register_admin_routes +from routes.assets import register_asset_routes +from routes.auth import register_auth_routes +from routes.pages import register_page_routes +from routes.settings import register_settings_routes from user_service import UserService -import json -# Load environment variables -load_dotenv() - -# Configure logging logging.basicConfig( level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.FileHandler('app.log'), - logging.StreamHandler() - ] + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + handlers=[logging.FileHandler("app.log"), logging.StreamHandler()], ) logger = logging.getLogger(__name__) -# Initialize Flask app -app = Flask(__name__, - static_folder='themes', - template_folder='templates') -app.config['SECRET_KEY'] = os.getenv('SECRET_KEY', 'dev-secret-key-change-in-production') -app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024 # 16MB max file size -app.config['ALLOW_ANONYMOUS_ACCESS'] = os.getenv('ALLOW_ANONYMOUS_ACCESS', 'true').lower() == 'true' -# Application branding configuration -app.config['APP_NAME'] = os.getenv('APP_NAME', 'BalanceBoard') -app.config['LOGO_PATH'] = os.getenv('LOGO_PATH', 'logo.png') +def create_app(config_class=Config): + """Create and configure the Flask application.""" + app = Flask(__name__, static_folder="themes", template_folder="templates") + app.config.from_object(config_class) -# Auth0 Configuration -app.config['AUTH0_DOMAIN'] = os.getenv('AUTH0_DOMAIN', '') -app.config['AUTH0_CLIENT_ID'] = os.getenv('AUTH0_CLIENT_ID', '') -app.config['AUTH0_CLIENT_SECRET'] = os.getenv('AUTH0_CLIENT_SECRET', '') -app.config['AUTH0_AUDIENCE'] = os.getenv('AUTH0_AUDIENCE', '') + init_db(app) + bcrypt.init_app(app) + login_manager.init_app(app) + oauth.init_app(app) -# Configuration constants -# Note: ALLOWED_FILTERSETS will be dynamically loaded from filter_engine -ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'} -UPLOAD_FOLDER = 'static/avatars' -MAX_FILENAME_LENGTH = 100 -DEFAULT_PORT = 5021 -DEFAULT_PAGE_SIZE = 20 -MIN_PASSWORD_LENGTH = 8 -MAX_USERNAME_LENGTH = 80 -MAX_EMAIL_LENGTH = 120 -MAX_COMMUNITY_NAME_LENGTH = 100 + user_service = UserService() + services_started = {"value": False} -# Initialize database -init_db(app) + @app.before_request + def _ensure_services_started(): + """Initialize background services lazily on the first request.""" + if services_started["value"]: + return -# Initialize bcrypt -bcrypt.init_app(app) - -# Initialize Flask-Login -login_manager = LoginManager() -login_manager.init_app(app) -login_manager.login_view = 'login' -login_manager.login_message = 'Please log in to access this page.' - -# Initialize user service -user_service = UserService() - -# Background services (polling scheduler + filter engine) are initialized -# lazily on the first request, NOT at import time. This keeps importing this -# module side-effect-free (no scheduler threads, no filter-engine init), so -# `from app import app` is safe for migration scripts and tests. The full -# app-factory split is deferred to Phase 1. -from polling_service import polling_service -from filter_pipeline import FilterEngine - -filter_engine = None # set lazily in _ensure_services_started() -_services_started = False - - -@app.before_request -def _ensure_services_started(): - """Start the polling scheduler and initialize the filter engine on the - first request, once. Kept out of module import so importing this module - has no side effects. - """ - global filter_engine, _services_started - if _services_started: - return - if filter_engine is None: - filter_engine = FilterEngine.get_instance() + filter_engine = get_filter_engine() logger.info( - f"FilterEngine initialized with " - f"{len(filter_engine.get_available_filtersets())} filtersets" + "FilterEngine initialized with %s filtersets", + len(filter_engine.get_available_filtersets()), ) - polling_service.init_app(app) - polling_service.start() - _services_started = True - -# Initialize OAuth for Auth0 -oauth = OAuth(app) -auth0 = oauth.register( - 'auth0', - client_id=app.config['AUTH0_CLIENT_ID'], - client_secret=app.config['AUTH0_CLIENT_SECRET'], - server_metadata_url=f'https://{app.config["AUTH0_DOMAIN"]}/.well-known/openid_configuration', - client_kwargs={ - 'scope': 'openid profile email', - } -) - -# Cache for posts and comments - improves performance -post_cache = {} -comment_cache = defaultdict(list) -cache_timestamp = 0 -CACHE_DURATION = 300 # 5 minutes - -# Security helper functions -def _is_safe_filterset(filterset): - """Validate filterset name for security""" - if not filterset or not isinstance(filterset, str): - return False - # filter_engine is initialized lazily on the first request; if it has not - # been initialized yet, fail closed. - if filter_engine is None: - return False - # Check against available filtersets from filter_engine - allowed_filtersets = set(filter_engine.get_available_filtersets()) - return filterset in allowed_filtersets and re.match(r'^[a-zA-Z0-9_-]+$', filterset) - -def _is_safe_path(path): - """Validate file path for security""" - if not path or not isinstance(path, str): - return False - # Check for directory traversal attempts - if '..' in path or path.startswith('/') or '\\' in path: - return False - # Only allow alphanumeric, dots, hyphens, underscores, and forward slashes - return re.match(r'^[a-zA-Z0-9._/-]+$', path) is not None - -def _is_allowed_file(filename): - """Check if file extension is allowed""" - return '.' in filename and \ - filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS - -def _load_posts_cache(): - """Load and cache posts data for better performance""" - global post_cache, comment_cache, cache_timestamp - - current_time = time.time() - if current_time - cache_timestamp < CACHE_DURATION and post_cache: - return post_cache, comment_cache - - # Clear existing cache - post_cache.clear() - comment_cache.clear() - - posts_dir = Path('data/posts') - comments_dir = Path('data/comments') - - # Load all posts - if posts_dir.exists(): - for post_file in posts_dir.glob('*.json'): - try: - with open(post_file, 'r') as f: - post_data = json.load(f) - post_uuid = post_data.get('uuid') - if post_uuid: - post_cache[post_uuid] = post_data - except (json.JSONDecodeError, IOError) as e: - logger.debug(f"Error reading post file {post_file}: {e}") - continue - - # Load all comments and group by post UUID - if comments_dir.exists(): - for comment_file in comments_dir.glob('*.json'): - try: - with open(comment_file, 'r') as f: - comment_data = json.load(f) - post_uuid = comment_data.get('post_uuid') - if post_uuid: - comment_cache[post_uuid].append(comment_data) - except (json.JSONDecodeError, IOError) as e: - logger.debug(f"Error reading comment file {comment_file}: {e}") - continue - - cache_timestamp = current_time - logger.info(f"Cache refreshed: {len(post_cache)} posts, {len(comment_cache)} comment groups") - return post_cache, comment_cache - -def _invalidate_cache(): - """Invalidate the cache to force refresh""" - global cache_timestamp - cache_timestamp = 0 - -def _validate_user_settings(settings_str): - """Validate and sanitize user settings JSON""" - try: - if not settings_str: - return {} - - settings = json.loads(settings_str) - if not isinstance(settings, dict): - logger.warning("User settings must be a JSON object") - return {} - - # Validate specific fields - validated = {} - - # Filter set validation - if 'filter_set' in settings: - filter_set = settings['filter_set'] - if isinstance(filter_set, str) and _is_safe_filterset(filter_set): - validated['filter_set'] = filter_set - - # Communities validation - if 'communities' in settings: - communities = settings['communities'] - if isinstance(communities, list): - # Validate each community name - safe_communities = [] - for community in communities: - if isinstance(community, str) and len(community) <= MAX_COMMUNITY_NAME_LENGTH and re.match(r'^[a-zA-Z0-9_-]+$', community): - safe_communities.append(community) - validated['communities'] = safe_communities - - # Experience settings validation - if 'experience' in settings: - exp = settings['experience'] - if isinstance(exp, dict): - safe_exp = {} - bool_fields = ['infinite_scroll', 'auto_refresh', 'push_notifications', 'dark_patterns_opt_in', 'time_filter_enabled'] - for field in bool_fields: - if field in exp and isinstance(exp[field], bool): - safe_exp[field] = exp[field] - - # Handle time_filter_days as integer - if 'time_filter_days' in exp and isinstance(exp['time_filter_days'], int) and exp['time_filter_days'] > 0: - safe_exp['time_filter_days'] = exp['time_filter_days'] - - validated['experience'] = safe_exp - - return validated - - except (json.JSONDecodeError, TypeError) as e: - logger.warning(f"Invalid user settings JSON: {e}") - return {} - -# Add custom Jinja filters -@app.template_filter('nl2br') -def nl2br_filter(text): - """Convert newlines to
tags. - - Escapes the input first (so raw HTML in user content cannot inject - markup), then inserts
tags and marks the result safe. Use this - instead of `| safe | nl2br`, which left an XSS hole. - """ - if not text: - return text - return Markup(str(escape(text)).replace('\n', '
\n')) - - -@app.template_filter('timeago') -def timeago_filter(timestamp): - """Format a unix timestamp as a relative time string ('3m ago', '5h ago', - '2d ago'), falling back to a date for older posts and 'Recently' for - missing/invalid input. Replaces the undefined `moment(...).fromNow()` - pattern that always rendered 'Recently'. - """ - if not timestamp: - return 'Recently' - try: - ts = float(timestamp) - except (TypeError, ValueError): - return 'Recently' - diff = time.time() - ts - if diff < 0: - return 'Recently' - if diff < 3600: - return f'{int(diff // 60)}m ago' - if diff < 86400: - return f'{int(diff // 3600)}h ago' - if diff < 604800: - return f'{int(diff // 86400)}d ago' - return time.strftime('%Y-%m-%d', time.localtime(ts)) - - -@login_manager.user_loader -def load_user(user_id): - """Load user by ID for Flask-Login""" - return user_service.get_user_by_id(user_id) - - -# ============================================================ -# STATIC CONTENT ROUTES -# ============================================================ - -@app.before_request -def check_first_user(): - """Check if any users exist, redirect to admin creation if not""" - # Skip for static files and auth routes - if request.endpoint and ( - request.endpoint.startswith('static') or - request.endpoint in ['login', 'signup', 'admin_setup', 'serve_theme', 'serve_logo'] - ): - return - - # Skip if user is already authenticated - if current_user.is_authenticated: - return - - # Check if any users exist - try: - user_count = User.query.count() - if user_count == 0: - return redirect(url_for('admin_setup')) - except Exception as e: - # If database is not ready, skip check - logger.warning(f"Database not ready for user count check: {e}") - pass - - -def calculate_quick_stats(): - """Calculate quick stats for dashboard""" - from datetime import datetime, timedelta - - cached_posts, _ = _load_posts_cache() - - # Calculate posts from today (last 24 hours) - now = datetime.utcnow() - today_start = now - timedelta(hours=24) - today_timestamp = today_start.timestamp() - - posts_today = sum(1 for post in cached_posts.values() - if post.get('timestamp', 0) >= today_timestamp) - - return { - 'posts_today': posts_today, - 'total_posts': len(cached_posts) - } - - -@app.route('/') -def index(): - """Serve the main feed page""" - # Calculate stats - quick_stats = calculate_quick_stats() - - if current_user.is_authenticated: - # Load user settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except (json.JSONDecodeError, TypeError) as e: - logger.warning(f"Invalid user settings JSON for user {current_user.id}: {e}") - user_settings = {} - - return render_template('dashboard.html', user_settings=user_settings, quick_stats=quick_stats) - else: - # Check if anonymous access is allowed - if app.config.get('ALLOW_ANONYMOUS_ACCESS', False): - # Anonymous mode - allow browsing with default settings - user_settings = { - 'filter_set': 'no_filter', - 'communities': [], - 'experience': { - 'infinite_scroll': False, - 'auto_refresh': False, - 'push_notifications': False, - 'dark_patterns_opt_in': False, - 'time_filter_enabled': False, - 'time_filter_days': 7 - } - } - return render_template('dashboard.html', user_settings=user_settings, anonymous=True, quick_stats=quick_stats) - else: - # Redirect non-authenticated users to login - return redirect(url_for('login')) - - -@app.route('/feed/') -def feed_content(filterset='no_filter'): - """Serve filtered feed content""" - # Validate filterset to prevent directory traversal - if not _is_safe_filterset(filterset): - logger.warning(f"Invalid filterset requested: {filterset}") - abort(404) - - # Additional path validation - safe_path = os.path.normpath(f'active_html/{filterset}/index.html') - if not safe_path.startswith('active_html/'): - logger.warning(f"Path traversal attempt detected: {filterset}") - abort(404) - - return send_from_directory(f'active_html/{filterset}', 'index.html') - -def load_platform_config(): - """Load platform configuration""" - try: - with open('platform_config.json', 'r') as f: - return json.load(f) - except (FileNotFoundError, json.JSONDecodeError, IOError) as e: - logger.warning(f"Could not load platform config: {e}") - return {"platforms": {}, "collection_targets": []} - - -def get_display_name_for_source(platform, source, platform_config): - """Get proper display name for a source based on platform""" - if not platform_config or 'platforms' not in platform_config: - return source - - platform_info = platform_config['platforms'].get(platform, {}) - - # For platforms with communities, find the community info - if platform_info.get('supports_communities'): - for community in platform_info.get('communities', []): - if community['id'] == source: - return community['display_name'] - # Fallback to prefix + source for Reddit-like platforms - prefix = platform_info.get('prefix', '') - return f"{prefix}{source}" if source else platform_info.get('name', platform) - else: - # For platforms without communities, use the platform name - return platform_info.get('name', platform) - - -@app.route('/api/posts') -def api_posts(): - """API endpoint to get posts data with pagination and filtering""" - try: - # Load platform configuration - platform_config = load_platform_config() - - # Get query parameters - page = int(request.args.get('page', 1)) - per_page = int(request.args.get('per_page', DEFAULT_PAGE_SIZE)) - community = request.args.get('community', '') - platform = request.args.get('platform', '') - search_query = request.args.get('q', '').lower().strip() - filter_override = request.args.get('filter', '') - - # Get user's filterset preference, community selections, and time filter - filterset_name = 'no_filter' - user_communities = [] - time_filter_enabled = False - time_filter_days = 7 - if current_user.is_authenticated: - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - filterset_name = user_settings.get('filter_set', 'no_filter') - user_communities = user_settings.get('communities', []) - - experience_settings = user_settings.get('experience', {}) - time_filter_enabled = experience_settings.get('time_filter_enabled', False) - time_filter_days = experience_settings.get('time_filter_days', 7) - except: - filterset_name = 'no_filter' - user_communities = [] - time_filter_enabled = False - time_filter_days = 7 - - # Override filterset if specified in request (for sidebar filter switching) - if filter_override and _is_safe_filterset(filter_override): - filterset_name = filter_override - - # Use cached data for better performance - cached_posts, cached_comments = _load_posts_cache() - - # Calculate time filter cutoff if enabled - time_cutoff = None - if time_filter_enabled: - from datetime import datetime, timedelta - cutoff_date = datetime.utcnow() - timedelta(days=time_filter_days) - time_cutoff = cutoff_date.timestamp() - - # Collect raw posts for filtering - raw_posts = [] - for post_uuid, post_data in cached_posts.items(): - # Apply time filter first if enabled - if time_filter_enabled and time_cutoff: - post_timestamp = post_data.get('timestamp', 0) - if post_timestamp < time_cutoff: - continue - # Apply community filter (before filterset) - if community and post_data.get('source', '').lower() != community.lower(): - continue - - # Apply platform filter (before filterset) - if platform and post_data.get('platform', '').lower() != platform.lower(): - continue - - # Apply user's community preferences (before filterset) - if user_communities: - post_source = post_data.get('source', '').lower() - post_platform = post_data.get('platform', '').lower() - post_id = post_data.get('id', '').lower() - - # Check if this post matches any of the user's selected communities - matches_community = False - for selected_community in user_communities: - selected_community = selected_community.lower() - - # Enhanced matching logic (platform-agnostic): - # 1. Exact source match (e.g., source="programming", community="programming") - # 2. Platform match (e.g., platform="hackernews", community="hackernews") - # 3. Partial match in source (e.g., source="programming", community="program") - # 4. Partial match in post ID (e.g., id="reddit_programming_123", community="programming") - if (post_source == selected_community or - post_platform == selected_community or - (selected_community in post_source) or - (selected_community in post_id)): - matches_community = True - break - - if not matches_community: - continue - - # Apply search filter (before filterset) - if search_query: - title = post_data.get('title', '').lower() - content = post_data.get('content', '').lower() - author = post_data.get('author', '').lower() - source = post_data.get('source', '').lower() - - if not (search_query in title or - search_query in content or - search_query in author or - search_query in source): - continue - - raw_posts.append(post_data) - - # Apply filterset using FilterEngine - filtered_posts = filter_engine.apply_filterset(raw_posts, filterset_name, use_cache=True) - - # Build response posts with metadata - posts = [] - for post_data in filtered_posts: - post_uuid = post_data.get('uuid') - comment_count = len(cached_comments.get(post_uuid, [])) - - # Get proper display name for source - source_display = get_display_name_for_source( - post_data.get('platform', ''), - post_data.get('source', ''), - platform_config - ) - - # Create post object with filter metadata - post = { - 'id': post_uuid, - 'title': post_data.get('title', 'Untitled'), - 'author': post_data.get('author', 'Unknown'), - 'platform': post_data.get('platform', 'unknown'), - 'score': post_data.get('score', 0), - 'timestamp': post_data.get('timestamp', 0), - 'url': f'/post/{post_uuid}', - 'comments_count': comment_count, - 'content_preview': (post_data.get('content', '') or '')[:200] + '...' if post_data.get('content') else '', - 'source': post_data.get('source', ''), - 'source_display': source_display, - 'tags': post_data.get('tags', []), - 'external_url': post_data.get('url', ''), - # Add filter metadata - 'filter_score': post_data.get('_filter_score', 0.5), - 'filter_categories': post_data.get('_filter_categories', []), - 'filter_tags': post_data.get('_filter_tags', []) - } - posts.append(post) - - # Sort by filter score (highest first), then timestamp - posts.sort(key=lambda x: (x['filter_score'], x['timestamp']), reverse=True) - - # Calculate pagination - total_posts = len(posts) - start_idx = (page - 1) * per_page - end_idx = start_idx + per_page - paginated_posts = posts[start_idx:end_idx] - - total_pages = (total_posts + per_page - 1) // per_page - has_next = page < total_pages - has_prev = page > 1 - - return { - 'posts': paginated_posts, - 'pagination': { - 'current_page': page, - 'total_pages': total_pages, - 'total_posts': total_posts, - 'per_page': per_page, - 'has_next': has_next, - 'has_prev': has_prev - } - } - - except Exception as e: - print(f"Error loading posts: {e}") - return {'posts': [], 'error': str(e), 'pagination': {'current_page': 1, 'total_pages': 0, 'total_posts': 0, 'per_page': DEFAULT_PAGE_SIZE, 'has_next': False, 'has_prev': False}} - - -@app.route('/api/platforms') -def api_platforms(): - """API endpoint to get platform configuration and available communities""" - try: - platform_config = load_platform_config() - - # Build community list for filtering UI - communities = [] - posts_dir = Path('data/posts') - source_counts = {} - - # Count posts per source to show actual available communities - for post_file in posts_dir.glob('*.json'): - try: - with open(post_file, 'r') as f: - post_data = json.load(f) - platform = post_data.get('platform', 'unknown') - source = post_data.get('source', '') - - key = f"{platform}:{source}" - source_counts[key] = source_counts.get(key, 0) + 1 - except: - continue - - # Build community list from actual data and platform config - for key, count in source_counts.items(): - platform, source = key.split(':', 1) - - # Get display info from platform config - platform_info = platform_config.get('platforms', {}).get(platform, {}) - community_info = None - - if platform_info.get('supports_communities'): - for community in platform_info.get('communities', []): - if community['id'] == source: - community_info = community - break - - # Create community entry - if community_info: - community_entry = { - 'platform': platform, - 'id': source, - 'name': community_info['name'], - 'display_name': community_info['display_name'], - 'icon': community_info.get('icon', platform_info.get('icon', '📄')), - 'count': count, - 'description': community_info.get('description', '') - } - else: - # Fallback for sources not in config - display_name = get_display_name_for_source(platform, source, platform_config) - community_entry = { - 'platform': platform, - 'id': source, - 'name': source or platform, - 'display_name': display_name, - 'icon': platform_info.get('icon', '📄'), - 'count': count, - 'description': f"Posts from {display_name}" - } - - communities.append(community_entry) - - # Sort communities by count (most posts first) - communities.sort(key=lambda x: x['count'], reverse=True) - - return { - 'platforms': platform_config.get('platforms', {}), - 'communities': communities, - 'total_communities': len(communities) - } - - except Exception as e: - print(f"Error loading platform configuration: {e}") - return { - 'platforms': {}, - 'communities': [], - 'total_communities': 0, - 'error': str(e) - } - - -@app.route('/api/content-timestamp') -def api_content_timestamp(): - """API endpoint to get the last content update timestamp for auto-refresh""" - try: - posts_dir = Path('data/posts') - - if not posts_dir.exists(): - return jsonify({'timestamp': 0}) - - # Get the most recent modification time of any post file - latest_mtime = 0 - for post_file in posts_dir.glob('*.json'): - mtime = post_file.stat().st_mtime - if mtime > latest_mtime: - latest_mtime = mtime - - return jsonify({'timestamp': latest_mtime}) - - except Exception as e: - logger.error(f"Error getting content timestamp: {e}") - return jsonify({'error': 'Failed to get content timestamp'}), 500 - - -@app.route('/api/bookmark', methods=['POST']) -@login_required -def api_bookmark(): - """Toggle bookmark status for a post""" - try: - from models import Bookmark - - data = request.get_json() - if not data or 'post_uuid' not in data: - return jsonify({'error': 'Missing post_uuid'}), 400 - - post_uuid = data['post_uuid'] - if not post_uuid: - return jsonify({'error': 'Invalid post_uuid'}), 400 - - # Check if bookmark already exists - existing_bookmark = Bookmark.query.filter_by( - user_id=current_user.id, - post_uuid=post_uuid - ).first() - - if existing_bookmark: - # Remove bookmark - db.session.delete(existing_bookmark) - db.session.commit() - return jsonify({'bookmarked': False, 'message': 'Bookmark removed'}) - else: - # Add bookmark - get post data for caching - cached_posts, _ = _load_posts_cache() - post_data = cached_posts.get(post_uuid, {}) - - bookmark = Bookmark( - user_id=current_user.id, - post_uuid=post_uuid, - title=post_data.get('title', ''), - platform=post_data.get('platform', ''), - source=post_data.get('source', '') - ) - db.session.add(bookmark) - db.session.commit() - return jsonify({'bookmarked': True, 'message': 'Bookmark added'}) - - except Exception as e: - logger.error(f"Error toggling bookmark: {e}") - return jsonify({'error': 'Failed to toggle bookmark'}), 500 - - -@app.route('/api/bookmarks') -@login_required -def api_bookmarks(): - """Get user's bookmarks""" - try: - from models import Bookmark - - page = int(request.args.get('page', 1)) - per_page = int(request.args.get('per_page', DEFAULT_PAGE_SIZE)) - - # Get user's bookmarks with pagination - bookmarks_query = Bookmark.query.filter_by(user_id=current_user.id).order_by(Bookmark.created_at.desc()) - total_bookmarks = bookmarks_query.count() - bookmarks = bookmarks_query.offset((page - 1) * per_page).limit(per_page).all() - - # Load current posts cache to get updated data - cached_posts, cached_comments = _load_posts_cache() - - # Build response - bookmark_posts = [] - for bookmark in bookmarks: - # Try to get current post data, fallback to cached data - post_data = cached_posts.get(bookmark.post_uuid) - if post_data: - # Post still exists in current data - comment_count = len(cached_comments.get(bookmark.post_uuid, [])) - post = { - 'id': bookmark.post_uuid, - 'title': post_data.get('title', bookmark.title or 'Untitled'), - 'author': post_data.get('author', 'Unknown'), - 'platform': post_data.get('platform', bookmark.platform or 'unknown'), - 'score': post_data.get('score', 0), - 'timestamp': post_data.get('timestamp', 0), - 'url': f'/post/{bookmark.post_uuid}', - 'comments_count': comment_count, - 'content_preview': (post_data.get('content', '') or '')[:200] + '...' if post_data.get('content') else '', - 'source': post_data.get('source', bookmark.source or ''), - 'bookmarked_at': bookmark.created_at.isoformat(), - 'external_url': post_data.get('url', '') - } - else: - # Post no longer in current data, use cached bookmark data - post = { - 'id': bookmark.post_uuid, - 'title': bookmark.title or 'Untitled', - 'author': 'Unknown', - 'platform': bookmark.platform or 'unknown', - 'score': 0, - 'timestamp': 0, - 'url': f'/post/{bookmark.post_uuid}', - 'comments_count': 0, - 'content_preview': 'Content no longer available', - 'source': bookmark.source or '', - 'bookmarked_at': bookmark.created_at.isoformat(), - 'external_url': '', - 'archived': True # Mark as archived - } - bookmark_posts.append(post) - - total_pages = (total_bookmarks + per_page - 1) // per_page - has_next = page < total_pages - has_prev = page > 1 - - return jsonify({ - 'posts': bookmark_posts, - 'pagination': { - 'current_page': page, - 'total_pages': total_pages, - 'total_posts': total_bookmarks, - 'per_page': per_page, - 'has_next': has_next, - 'has_prev': has_prev - } - }) - - except Exception as e: - logger.error(f"Error getting bookmarks: {e}") - return jsonify({'error': 'Failed to get bookmarks'}), 500 - - -@app.route('/api/bookmark-status/') -@login_required -def api_bookmark_status(post_uuid): - """Check if a post is bookmarked by current user""" - try: - from models import Bookmark - - bookmark = Bookmark.query.filter_by( - user_id=current_user.id, - post_uuid=post_uuid - ).first() - - return jsonify({'bookmarked': bookmark is not None}) - - except Exception as e: - logger.error(f"Error checking bookmark status: {e}") - return jsonify({'error': 'Failed to check bookmark status'}), 500 - - -@app.route('/api/filters') -def api_filters(): - """API endpoint to get available filters""" - try: - filters = [] - - # Get current user's filter preference - current_filter = 'no_filter' - if current_user.is_authenticated: - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - current_filter = user_settings.get('filter_set', 'no_filter') - except: - pass - - # Get available filtersets from filter engine - for filterset_name in filter_engine.get_available_filtersets(): - filterset_config = filter_engine.config.get_filterset(filterset_name) - if filterset_config: - # Map filter names to icons and display names - icon_map = { - 'no_filter': '🌐', - 'safe_content': '✅', - 'tech_only': '💻', - 'high_quality': '⭐', - 'custom_example': '🎯' - } - - name_map = { - 'no_filter': 'All Content', - 'safe_content': 'Safe Content', - 'tech_only': 'Tech Only', - 'high_quality': 'High Quality', - 'custom_example': 'Custom Example' - } - - filters.append({ - 'id': filterset_name, - 'name': name_map.get(filterset_name, filterset_name.replace('_', ' ').title()), - 'description': filterset_config.get('description', ''), - 'icon': icon_map.get(filterset_name, '🔧'), - 'active': filterset_name == current_filter - }) - - return jsonify({'filters': filters}) - - except Exception as e: - logger.error(f"Error getting filters: {e}") - return jsonify({'error': 'Failed to get filters'}), 500 - - -@app.route('/bookmarks') -@login_required -def bookmarks(): - """Bookmarks page""" - return render_template('bookmarks.html', user=current_user) - - -def build_comment_tree(comments): - """Build a hierarchical comment tree from flat comment list""" - # Create lookup dict by UUID - comment_dict = {c['uuid']: {**c, 'replies': []} for c in comments} - - # Build tree structure - root_comments = [] - for comment in comments: - parent_uuid = comment.get('parent_comment_uuid') - if parent_uuid and parent_uuid in comment_dict: - # Add as reply to parent - comment_dict[parent_uuid]['replies'].append(comment_dict[comment['uuid']]) - else: - # Top-level comment - root_comments.append(comment_dict[comment['uuid']]) - - # Sort at each level by timestamp - def sort_tree(comments_list): - comments_list.sort(key=lambda x: x.get('timestamp', 0)) - for comment in comments_list: - if comment.get('replies'): - sort_tree(comment['replies']) - - sort_tree(root_comments) - return root_comments - - -@app.route('/post/') -def post_detail(post_id): - """Serve individual post detail page with modern theme""" - try: - # Load platform configuration - platform_config = load_platform_config() - - # Use cached data for better performance - cached_posts, cached_comments = _load_posts_cache() - - # Get post data from cache - post_data = cached_posts.get(post_id) - if not post_data: - return render_template('404.html'), 404 - - # Add source display name - post_data['source_display'] = get_display_name_for_source( - post_data.get('platform', ''), - post_data.get('source', ''), - platform_config - ) - - # Get comments from cache - comments_flat = cached_comments.get(post_id, []) - logger.info(f"Loading post {post_id}: found {len(comments_flat)} comments") - - # Build comment tree - comments = build_comment_tree(comments_flat) - - # Load user settings if authenticated - user_settings = {} - if current_user.is_authenticated: - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except: - user_settings = {} - - return render_template('post_detail.html', post=post_data, comments=comments, user_settings=user_settings) - - except Exception as e: - print(f"Error loading post {post_id}: {e}") - return render_template('404.html'), 404 - - -@app.route('/themes/') -def serve_theme(filename): - """Serve theme files (CSS, JS)""" - # Validate filename to prevent directory traversal - if not _is_safe_path(filename) or '..' in filename: - logger.warning(f"Unsafe theme file requested: {filename}") - abort(404) - return send_from_directory('themes', filename) - - -@app.route('/logo.png') -def serve_logo(): - """Serve configurable logo""" - logo_path = app.config['LOGO_PATH'] - # If it's just a filename, serve from current directory - if '/' not in logo_path: - return send_from_directory('.', logo_path) - else: - # If it's a full path, split directory and filename - directory = os.path.dirname(logo_path) - filename = os.path.basename(logo_path) - return send_from_directory(directory, filename) - -@app.route('/static/') -def serve_static(filename): - """Serve static files (avatars, etc.)""" - # Validate filename to prevent directory traversal - if not _is_safe_path(filename) or '..' in filename: - logger.warning(f"Unsafe static file requested: {filename}") - abort(404) - return send_from_directory('static', filename) - - -# ============================================================ -# AUTHENTICATION ROUTES -# ============================================================ - -@app.route('/login', methods=['GET', 'POST']) -def login(): - """Login page""" - if current_user.is_authenticated: - return redirect(url_for('index')) - - # Check if Auth0 is configured - auth0_configured = bool(app.config.get('AUTH0_DOMAIN') and app.config.get('AUTH0_CLIENT_ID')) - - if request.method == 'POST': - username = request.form.get('username') - password = request.form.get('password') - remember = request.form.get('remember', False) == 'on' - - if not user_service: - flash('User service not available', 'error') - return render_template('login.html', auth0_configured=auth0_configured) - - user = user_service.authenticate(username, password) - - if user: - login_user(user, remember=remember) - flash(f'Welcome back, {user.username}!', 'success') - - # Redirect to next page or home - next_page = request.args.get('next') - return redirect(next_page) if next_page else redirect(url_for('index')) - else: - flash('Invalid username or password', 'error') - - return render_template('login.html', auth0_configured=auth0_configured) - - -@app.route('/password-reset-request', methods=['GET', 'POST']) -def password_reset_request(): - """Request a password reset""" - if current_user.is_authenticated: - return redirect(url_for('index')) - - if request.method == 'POST': - email = request.form.get('email', '').strip().lower() - - if not email: - flash('Please enter your email address', 'error') - return render_template('password_reset_request.html') - - # Find user by email - user = User.query.filter_by(email=email).first() - - # Always show success message for security (don't reveal if email exists) - flash('If an account exists with that email, a password reset link has been sent.', 'success') - - if user and user.password_hash: # Only send reset if user has a password (not OAuth only) - # Generate reset token - token = user.generate_reset_token() - - # Build reset URL - reset_url = url_for('password_reset', token=token, _external=True) - - # Log the reset URL (in production, this would be emailed) - logger.info(f"Password reset requested for {email}. Reset URL: {reset_url}") - - # For now, also flash it for development (remove in production) - flash(f'Reset link (development only): {reset_url}', 'info') - - return redirect(url_for('login')) - - return render_template('password_reset_request.html') - - -@app.route('/password-reset/', methods=['GET', 'POST']) -def password_reset(token): - """Reset password with token""" - if current_user.is_authenticated: - return redirect(url_for('index')) - - # Find user by token - user = User.query.filter_by(reset_token=token).first() - - if not user or not user.verify_reset_token(token): - flash('Invalid or expired reset token', 'error') - return redirect(url_for('login')) - - if request.method == 'POST': - password = request.form.get('password', '') - confirm_password = request.form.get('confirm_password', '') - - if not password or len(password) < MIN_PASSWORD_LENGTH: - flash(f'Password must be at least {MIN_PASSWORD_LENGTH} characters', 'error') - return render_template('password_reset.html') - - if password != confirm_password: - flash('Passwords do not match', 'error') - return render_template('password_reset.html') - - # Set new password - user.set_password(password) - user.clear_reset_token() - - flash('Your password has been reset successfully. You can now log in.', 'success') - return redirect(url_for('login')) - - return render_template('password_reset.html') - - -# Auth0 Routes -@app.route('/auth0/login') -def auth0_login(): - """Redirect to Auth0 for authentication""" - # Check if Auth0 is configured - if not app.config.get('AUTH0_DOMAIN') or not app.config.get('AUTH0_CLIENT_ID'): - flash('Auth0 authentication is not configured. Please use email/password login or contact the administrator.', 'error') - return redirect(url_for('login')) - - try: - redirect_uri = url_for('auth0_callback', _external=True) - return auth0.authorize_redirect(redirect_uri) - except Exception as e: - logger.error(f"Auth0 login error: {e}") - flash('Auth0 authentication failed. Please use email/password login.', 'error') - return redirect(url_for('login')) - - -@app.route('/auth0/callback') -def auth0_callback(): - """Handle Auth0 callback and create/login user""" - try: - # Get the access token from Auth0 - token = auth0.authorize_access_token() - - # Get user info from Auth0 - user_info = token.get('userinfo') - if not user_info: - user_info = auth0.parse_id_token(token) - - # Extract user details - auth0_id = user_info.get('sub') - email = user_info.get('email') - username = user_info.get('nickname') or user_info.get('preferred_username') or email.split('@')[0] - - if not auth0_id or not email: - flash('Unable to get user information from Auth0', 'error') - return redirect(url_for('login')) - - # Check if user exists with this Auth0 ID - user = user_service.get_user_by_auth0_id(auth0_id) - - if not user: - # Check if user exists with this email (for account linking) - existing_user = user_service.get_user_by_email(email) - - if existing_user: - # Link existing account to Auth0 - user_service.link_auth0_account(existing_user.id, auth0_id) - user = existing_user - flash(f'Account linked successfully! Welcome back, {user.username}!', 'success') - else: - # Create new user - # Generate unique username if needed - base_username = username[:MAX_USERNAME_LENGTH-3] # Leave room for suffix - unique_username = base_username - counter = 1 - while user_service.username_exists(unique_username): - unique_username = f"{base_username}_{counter}" - counter += 1 - - user_id = user_service.create_user( - username=unique_username, - email=email, - password=None, # No password for OAuth users - is_admin=False, - auth0_id=auth0_id - ) - - if user_id: - user = user_service.get_user_by_id(user_id) - flash(f'Account created successfully! Welcome, {user.username}!', 'success') - else: - flash('Failed to create user account', 'error') - return redirect(url_for('login')) - else: - flash(f'Welcome back, {user.username}!', 'success') - - # Log in the user - if user: - login_user(user, remember=True) - - # Store Auth0 info in session for future use - session['auth0_user_info'] = user_info - - # Redirect to next page or home - next_page = request.args.get('next') - return redirect(next_page) if next_page else redirect(url_for('index')) - - except Exception as e: - logger.error(f"Auth0 callback error: {e}") - flash('Authentication failed. Please try again.', 'error') - - return redirect(url_for('login')) - - -@app.route('/auth0/logout') -@login_required -def auth0_logout(): - """Logout from Auth0 and local session""" - # Clear session - session.clear() - logout_user() - - # Build Auth0 logout URL - domain = app.config['AUTH0_DOMAIN'] - client_id = app.config['AUTH0_CLIENT_ID'] - return_to = url_for('index', _external=True) - - logout_url = f'https://{domain}/v2/logout?' + urlencode({ - 'returnTo': return_to, - 'client_id': client_id - }, quote_via=quote_plus) - - return redirect(logout_url) - - -@app.route('/admin-setup', methods=['GET', 'POST']) -def admin_setup(): - """Create first admin user""" - # Check if users already exist - try: - user_count = User.query.count() - if user_count > 0: - flash('Admin user already exists.', 'info') - return redirect(url_for('login')) - except Exception as e: - logger.warning(f"Database error checking existing users: {e}") - pass - - if request.method == 'POST': - username = request.form.get('username') - email = request.form.get('email') - password = request.form.get('password') - password_confirm = request.form.get('password_confirm') - - # Validation - if not username or not email or not password: - flash('All fields are required', 'error') - return render_template('admin_setup.html') - - if password != password_confirm: - flash('Passwords do not match', 'error') - return render_template('admin_setup.html') - - if len(password) < MIN_PASSWORD_LENGTH: - flash(f'Password must be at least {MIN_PASSWORD_LENGTH} characters', 'error') - return render_template('admin_setup.html') - - # Create admin user - user_id = user_service.create_user(username, email, password, is_admin=True) - - if user_id: - flash('Admin account created successfully! Please log in.', 'success') - return redirect(url_for('login')) - else: - flash('Error creating admin account. Please try again.', 'error') - - return render_template('admin_setup.html') - - -@app.route('/signup', methods=['GET', 'POST']) -def signup(): - """Signup page""" - if current_user.is_authenticated: - return redirect(url_for('index')) - - if request.method == 'POST': - username = request.form.get('username') - email = request.form.get('email') - password = request.form.get('password') - password_confirm = request.form.get('password_confirm') - - if not user_service: - flash('User service not available', 'error') - return render_template('signup.html') - - # Validation - if not username or not email or not password: - flash('All fields are required', 'error') - return render_template('signup.html') - - if password != password_confirm: - flash('Passwords do not match', 'error') - return render_template('signup.html') - - if len(password) < MIN_PASSWORD_LENGTH: - flash(f'Password must be at least {MIN_PASSWORD_LENGTH} characters', 'error') - return render_template('signup.html') - - if user_service.username_exists(username): - flash('Username already taken', 'error') - return render_template('signup.html') - - if user_service.email_exists(email): - flash('Email already registered', 'error') - return render_template('signup.html') - - # Create user - user_id = user_service.create_user(username, email, password) - - if user_id: - flash('Account created successfully! Please log in.', 'success') - return redirect(url_for('login')) - else: - flash('Error creating account. Please try again.', 'error') - - return render_template('signup.html') - - -@app.route('/logout') -@login_required -def logout(): - """Logout current user""" - logout_user() - flash('You have been logged out.', 'info') - return redirect(url_for('index')) - - -@app.route('/settings') -@login_required -def settings(): - """Main settings page""" - # Load user settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except: - user_settings = {} - - # Load available filter sets - try: - with open('filtersets.json', 'r') as f: - filter_sets = json.load(f) - except: - filter_sets = {} - - return render_template('settings.html', - user=current_user, - user_settings=user_settings, - filter_sets=filter_sets) - - -@app.route('/settings/profile', methods=['GET', 'POST']) -@login_required -def settings_profile(): - """Profile settings page""" - if request.method == 'POST': - username = request.form.get('username') - email = request.form.get('email') - default_avatar = request.form.get('default_avatar') - - # Validation - if not username or not email: - flash('Username and email are required', 'error') - return render_template('settings_profile.html', user=current_user) - - # Check if username is taken by another user - if username != current_user.username and user_service.username_exists(username): - flash('Username already taken', 'error') - return render_template('settings_profile.html', user=current_user) - - # Check if email is taken by another user - if email != current_user.email and user_service.email_exists(email): - flash('Email already registered', 'error') - return render_template('settings_profile.html', user=current_user) - - # Update user - current_user.username = username - current_user.email = email - - # Handle default avatar selection - if default_avatar and default_avatar.startswith('default_'): - current_user.profile_picture_url = f"/static/default-avatars/{default_avatar}.png" - - db.session.commit() - - flash('Profile updated successfully', 'success') - return redirect(url_for('settings')) - - # Available default avatars - default_avatars = [ - {'id': 'default_1', 'name': 'Gradient Blue', 'bg': 'linear-gradient(135deg, #667eea 0%, #764ba2 100%)'}, - {'id': 'default_2', 'name': 'Gradient Green', 'bg': 'linear-gradient(135deg, #4facfe 0%, #00f2fe 100%)'}, - {'id': 'default_3', 'name': 'Gradient Orange', 'bg': 'linear-gradient(135deg, #fa709a 0%, #fee140 100%)'}, - {'id': 'default_4', 'name': 'Gradient Purple', 'bg': 'linear-gradient(135deg, #a8edea 0%, #fed6e3 100%)'}, - {'id': 'default_5', 'name': 'Brand Colors', 'bg': 'linear-gradient(135deg, #4db6ac 0%, #26a69a 100%)'}, - {'id': 'default_6', 'name': 'Sunset', 'bg': 'linear-gradient(135deg, #ff7e5f 0%, #feb47b 100%)'}, - ] - - return render_template('settings_profile.html', user=current_user, default_avatars=default_avatars) - - -@app.route('/settings/communities', methods=['GET', 'POST']) -@login_required -def settings_communities(): - """Community/source selection settings""" - if request.method == 'POST': - # Get selected communities - selected_communities = request.form.getlist('communities') - - # Load current settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except: - user_settings = {} - - # Update communities - user_settings['communities'] = selected_communities - - # Save settings - current_user.settings = json.dumps(user_settings) - db.session.commit() - - flash('Community preferences updated', 'success') - return redirect(url_for('settings')) - - # Load current settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - selected_communities = user_settings.get('communities', []) - except: - selected_communities = [] - - # Get available communities from platform config and collection targets - available_communities = [] - - # Load platform configuration with error handling - try: - platform_config = load_platform_config() - if not platform_config: - platform_config = {"platforms": {}, "collection_targets": []} - except Exception as e: - logger.error(f"Error loading platform config: {e}") - platform_config = {"platforms": {}, "collection_targets": []} - - # Get enabled communities from collection_targets (what's actually being crawled) - enabled_communities = set() - try: - for target in platform_config.get('collection_targets', []): - if 'platform' in target and 'community' in target: - enabled_communities.add((target['platform'], target['community'])) - except Exception as e: - logger.error(f"Error processing collection_targets: {e}") - - # Build community list from platform config for communities that are enabled - try: - for platform_name, platform_info in platform_config.get('platforms', {}).items(): - if not isinstance(platform_info, dict): - continue - communities = platform_info.get('communities', []) - if not isinstance(communities, list): - continue - - for community_info in communities: - try: - if not isinstance(community_info, dict): - continue - - # Only include communities that are in collection_targets - if (platform_name, community_info['id']) in enabled_communities: - available_communities.append({ - 'id': community_info['id'], - 'name': community_info['name'], - 'display_name': community_info.get('display_name', community_info['name']), - 'platform': platform_name, - 'icon': community_info.get('icon', platform_info.get('icon', '📄')), - 'description': community_info.get('description', '') - }) - except Exception as e: - logger.error(f"Error processing community {community_info}: {e}") - continue - except Exception as e: - logger.error(f"Error building community list: {e}") - - logger.info(f"Found {len(available_communities)} available communities") - - return render_template('settings_communities.html', - user=current_user, - available_communities=available_communities, - selected_communities=selected_communities) - - -@app.route('/settings/filters', methods=['GET', 'POST']) -@login_required -def settings_filters(): - """Filter settings page""" - if request.method == 'POST': - selected_filter = request.form.get('filter_set', 'no_filter') - - # Load and validate current settings - user_settings = _validate_user_settings(current_user.settings) - - # Validate new filter setting - if _is_safe_filterset(selected_filter): - user_settings['filter_set'] = selected_filter - else: - flash('Invalid filter selection', 'error') - return redirect(url_for('settings')) - - # Save validated settings - try: - current_user.settings = json.dumps(user_settings) - db.session.commit() - flash('Filter settings updated successfully', 'success') - except Exception as e: - db.session.rollback() - logger.error(f"Error saving filter settings for user {current_user.id}: {e}") - flash('Error saving settings', 'error') - - return redirect(url_for('settings')) - - # Load current settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except: - user_settings = {} - - current_filter = user_settings.get('filter_set', 'no_filter') - - # Load available filter sets from FilterEngine as a dictionary - filter_sets = {} - for filterset_name in filter_engine.get_available_filtersets(): - filter_sets[filterset_name] = filter_engine.config.get_filterset(filterset_name) - - return render_template('settings_filters.html', - user=current_user, - filter_sets=filter_sets, - current_filter=current_filter) - - -@app.route('/settings/experience', methods=['GET', 'POST']) -@login_required -def settings_experience(): - """Experience and behavioral settings page - opt-in addictive features""" - if request.method == 'POST': - # Load current settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except: - user_settings = {} - - # Get experience settings with defaults (all opt-in, so default to False) - user_settings['experience'] = { - 'infinite_scroll': request.form.get('infinite_scroll') == 'on', - 'auto_refresh': request.form.get('auto_refresh') == 'on', - 'push_notifications': request.form.get('push_notifications') == 'on', - 'dark_patterns_opt_in': request.form.get('dark_patterns_opt_in') == 'on', - 'time_filter_enabled': request.form.get('time_filter_enabled') == 'on', - 'time_filter_days': int(request.form.get('time_filter_days', 7)) - } - - # Save settings - current_user.settings = json.dumps(user_settings) - db.session.commit() - - flash('Experience settings updated successfully', 'success') - return redirect(url_for('settings')) - - # Load current settings - try: - user_settings = json.loads(current_user.settings) if current_user.settings else {} - except: - user_settings = {} - - experience_settings = user_settings.get('experience', { - 'infinite_scroll': False, - 'auto_refresh': False, - 'push_notifications': False, - 'dark_patterns_opt_in': False, - 'time_filter_enabled': False, - 'time_filter_days': 7 - }) - - return render_template('settings_experience.html', - user=current_user, - experience_settings=experience_settings) - - -@app.route('/upload-avatar', methods=['POST']) -@login_required -def upload_avatar(): - """Upload profile picture""" - try: - # Debug logging - logger.info(f"Avatar upload attempt by user {current_user.id} ({current_user.username})") - logger.debug(f"Request files: {list(request.files.keys())}") - logger.debug(f"Request form: {dict(request.form)}") - - # Check if user is properly authenticated and has required attributes - if not hasattr(current_user, 'id') or not current_user.id: - logger.error("User missing ID attribute") - flash('Authentication error. Please log in again.', 'error') - return redirect(url_for('login')) - - if not hasattr(current_user, 'username') or not current_user.username: - logger.error("User missing username attribute") - flash('User profile incomplete. Please update your profile.', 'error') - return redirect(url_for('settings_profile')) - - # Check for file in request - if 'avatar' not in request.files: - logger.warning("No avatar file in request") - flash('No file selected', 'error') - return redirect(url_for('settings_profile')) - - file = request.files['avatar'] - if file.filename == '': - logger.warning("Empty filename provided") - flash('No file selected', 'error') - return redirect(url_for('settings_profile')) - - logger.info(f"Processing file: {file.filename}") - - # Validate file type and size - if not _is_allowed_file(file.filename): - logger.warning(f"Invalid file type: {file.filename}") - flash('Invalid file type. Please upload PNG, JPG, or GIF', 'error') - return redirect(url_for('settings_profile')) - - # Check file size (Flask's MAX_CONTENT_LENGTH handles this too, but double-check) - if hasattr(file, 'content_length') and file.content_length > app.config.get('MAX_CONTENT_LENGTH', 16*1024*1024): - logger.warning(f"File too large: {file.content_length}") - flash('File too large. Maximum size is 16MB', 'error') - return redirect(url_for('settings_profile')) - - # Validate and secure filename - filename = secure_filename(file.filename) - if not filename or len(filename) > MAX_FILENAME_LENGTH: - logger.warning(f"Invalid filename after sanitization: {filename}") - flash('Invalid filename', 'error') - return redirect(url_for('settings_profile')) - - # Add user ID to make filename unique and prevent conflicts - unique_filename = f"{current_user.id}_{filename}" - logger.info(f"Generated unique filename: {unique_filename}") - - # Ensure upload directory exists and is secure - upload_dir = os.path.abspath(UPLOAD_FOLDER) - os.makedirs(upload_dir, exist_ok=True) - - upload_path = os.path.join(upload_dir, unique_filename) - - # Final security check - ensure path is within upload directory - if not os.path.abspath(upload_path).startswith(upload_dir): - logger.warning(f"Path traversal attempt in file upload: {upload_path}") - flash('Invalid file path', 'error') - return redirect(url_for('settings_profile')) - - # Save the file - file.save(upload_path) - logger.info(f"File saved successfully: {upload_path}") - - # Update user profile with database error handling - old_avatar_url = current_user.profile_picture_url - current_user.profile_picture_url = f"/static/avatars/{unique_filename}" - - db.session.commit() - logger.info(f"User profile updated successfully for {current_user.username}") - - # Clean up old avatar file if it exists and was uploaded by user - if old_avatar_url and old_avatar_url.startswith('/static/avatars/') and current_user.id in old_avatar_url: - try: - old_file_path = os.path.join(upload_dir, os.path.basename(old_avatar_url)) - if os.path.exists(old_file_path): - os.remove(old_file_path) - logger.info(f"Cleaned up old avatar: {old_file_path}") - except Exception as e: - logger.warning(f"Could not clean up old avatar: {e}") - - flash('Profile picture updated successfully', 'success') - return redirect(url_for('settings_profile')) - - except Exception as e: - logger.error(f"Unexpected error in avatar upload: {e}") - db.session.rollback() - flash('An unexpected error occurred. Please try again.', 'error') - return redirect(url_for('settings_profile')) - - -@app.route('/profile') -@login_required -def profile(): - """User profile page""" - return render_template('profile.html', user=current_user) - - -# ============================================================ -# ADMIN ROUTES -# ============================================================ - -@app.route('/admin') -@login_required -def admin_panel(): - """Admin panel - user management""" - if not current_user.is_admin: - flash('Access denied. Admin privileges required.', 'error') - return redirect(url_for('index')) - - if not user_service: - flash('User service not available', 'error') - return redirect(url_for('index')) - - users = user_service.get_all_users() - return render_template('admin.html', users=users) - - -@app.route('/admin/user//delete', methods=['POST']) -@login_required -def admin_delete_user(user_id): - """Delete user (admin only)""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) - - # Prevent self-deletion - if current_user.id == user_id: - flash('You cannot delete your own account!', 'error') - return redirect(url_for('admin_panel')) - - user = user_service.get_user_by_id(user_id) - if user: - username = user.username - if user_service.delete_user(user_id): - flash(f'User {username} has been deleted.', 'success') - logger.info(f"Admin {current_user.id} deleted user {username} ({user_id})") - else: - flash('Error deleting user', 'error') - logger.error(f"Failed to delete user {user_id}") - else: - flash('User not found', 'error') - - return redirect(url_for('admin_panel')) - - -@app.route('/admin/user//toggle-admin', methods=['POST']) -@login_required -def admin_toggle_admin(user_id): - """Toggle user admin status""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) - - target_user = user_service.get_user_by_id(user_id) - - if target_user: - new_status = not target_user.is_admin # Toggle admin status - user_service.update_user_admin_status(user_id, new_status) - flash('Admin status updated', 'success') - else: - flash('User not found', 'error') - - return redirect(url_for('admin_panel')) - - -# This route is duplicate - removed in favor of the UUID-based route above - - -# This route is duplicate - removed in favor of the UUID-based route above - - -@app.route('/admin/regenerate_content', methods=['POST']) -@login_required -def admin_regenerate_content(): - """Regenerate all HTML content""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('admin_panel')) - - try: - import subprocess - import shlex - - # Secure subprocess execution with absolute paths and validation - script_path = os.path.abspath('generate_html.py') - if not os.path.exists(script_path): - flash('Content generation script not found', 'error') - return redirect(url_for('admin_panel')) - - # Use absolute python path and validate arguments - python_exe = os.path.abspath(os.sys.executable) - cmd = [python_exe, script_path, '--filterset', 'no_filter', '--theme', 'vanilla-js'] - - # Execute with timeout and security restrictions - result = subprocess.run( - cmd, - capture_output=True, - text=True, - cwd=os.path.abspath('.'), - timeout=300, # 5 minute timeout - check=False - ) - - if result.returncode == 0: - flash('Content regenerated successfully', 'success') - logger.info(f"Content regenerated by admin user {current_user.id}") - # Invalidate cache since content was regenerated - _invalidate_cache() - else: - flash('Error regenerating content', 'error') - logger.error(f"Content regeneration failed: {result.stderr}") - - except subprocess.TimeoutExpired: - flash('Content regeneration timed out', 'error') - logger.error("Content regeneration timed out") - except Exception as e: - flash(f'Error regenerating content: {str(e)}', 'error') - logger.error(f"Content regeneration error: {e}") - - return redirect(url_for('admin_panel')) - - -@app.route('/admin/clear_cache', methods=['POST']) -@login_required -def admin_clear_cache(): - """Clear application cache""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('admin_panel')) - - try: - # Clear any cache directories or temp files - import shutil - import os - - cache_dirs = ['cache', 'temp'] - for cache_dir in cache_dirs: - if os.path.exists(cache_dir): - shutil.rmtree(cache_dir) - - # Clear application cache - _invalidate_cache() - - flash('Cache cleared successfully', 'success') - logger.info(f"Cache cleared by admin user {current_user.id}") - except Exception as e: - flash(f'Error clearing cache: {str(e)}', 'error') - logger.error(f"Cache clearing error: {e}") - - return redirect(url_for('admin_panel')) - - -@app.route('/admin/backup_data', methods=['POST']) -@login_required -def admin_backup_data(): - """Create backup of application data""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('admin_panel')) - - try: - import shutil - import os - from datetime import datetime - - timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') - backup_name = f'balanceboard_backup_{timestamp}' - - # Create backup directory - backup_dir = f'backups/{backup_name}' - os.makedirs(backup_dir, exist_ok=True) - - # Copy important directories - dirs_to_backup = ['data', 'templates', 'themes', 'static'] - for dir_name in dirs_to_backup: - if os.path.exists(dir_name): - shutil.copytree(dir_name, f'{backup_dir}/{dir_name}') - - # Copy important files - files_to_backup = ['app.py', 'models.py', 'database.py', 'filtersets.json'] - for file_name in files_to_backup: - if os.path.exists(file_name): - shutil.copy2(file_name, backup_dir) - - flash(f'Backup created: {backup_name}', 'success') - except Exception as e: - flash(f'Error creating backup: {str(e)}', 'error') - - return redirect(url_for('admin_panel')) - - -# ============================================================ -# POLLING MANAGEMENT ROUTES -# ============================================================ - -@app.route('/admin/polling') -@login_required -def admin_polling(): - """Admin polling management page""" - if not current_user.is_admin: - flash('Access denied. Admin privileges required.', 'error') - return redirect(url_for('index')) - - from models import PollSource, PollLog - from polling_service import polling_service - - # Get all poll sources with recent logs - sources = PollSource.query.order_by(PollSource.platform, PollSource.display_name).all() - - # Get scheduler status - scheduler_status = polling_service.get_status() - - # Load platform config for available sources - platform_config = load_platform_config() - - return render_template('admin_polling.html', - sources=sources, - scheduler_status=scheduler_status, - platform_config=platform_config) - - -@app.route('/admin/polling/add', methods=['POST']) -@login_required -def admin_polling_add(): - """Add a new poll source""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) - - from models import PollSource - - platform = request.form.get('platform') - source_id = request.form.get('source_id') - custom_source_id = request.form.get('custom_source_id') - display_name = request.form.get('display_name') - poll_interval = int(request.form.get('poll_interval', 60)) - max_posts = int(request.form.get('max_posts', 100)) - fetch_comments = request.form.get('fetch_comments', 'true') == 'true' - priority = request.form.get('priority', 'medium') - - # Use custom source if provided, otherwise use dropdown - if custom_source_id and custom_source_id.strip(): - source_id = custom_source_id.strip() - - if not platform or not source_id or not display_name: - flash('Missing required fields', 'error') - return redirect(url_for('admin_polling')) - - # Check if source already exists - existing = PollSource.query.filter_by(platform=platform, source_id=source_id).first() - if existing: - flash(f'Source {platform}:{source_id} already exists', 'warning') - return redirect(url_for('admin_polling')) - - # Create new source (disabled by default) - source = PollSource( - platform=platform, - source_id=source_id, - display_name=display_name, - poll_interval_minutes=poll_interval, - max_posts=max_posts, - fetch_comments=fetch_comments, - priority=priority, - enabled=False, - created_by=current_user.id + polling_service = get_polling_service() + polling_service.init_app(app) + polling_service.start() + services_started["value"] = True + + auth0 = oauth.register( + "auth0", + client_id=app.config["AUTH0_CLIENT_ID"], + client_secret=app.config["AUTH0_CLIENT_SECRET"], + server_metadata_url=f"https://{app.config['AUTH0_DOMAIN']}/.well-known/openid_configuration", + client_kwargs={"scope": "openid profile email"}, ) - db.session.add(source) - db.session.commit() + app.register_blueprint(create_api_blueprint(), url_prefix="/api/v1") + register_asset_routes(app) + register_page_routes(app) + register_settings_routes(app, user_service) + register_admin_routes(app, user_service) + register_auth_routes(app, user_service, auth0) - flash(f'Added polling source: {display_name}', 'success') - logger.info(f"Admin {current_user.id} added poll source {platform}:{source_id}") + register_template_filters(app) + register_login_loader(user_service) + register_request_guards(app) + register_context_processors(app) + register_error_handlers(app) - return redirect(url_for('admin_polling')) + return app -@app.route('/admin/polling//toggle', methods=['POST']) -@login_required -def admin_polling_toggle(source_id): - """Toggle a poll source on/off""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) +def register_template_filters(app): + """Register Jinja filters used by legacy templates.""" - from models import PollSource + @app.template_filter("nl2br") + def nl2br_filter(text): + """Convert newlines to escaped
tags.""" + if not text: + return text + return Markup(str(escape(text)).replace("\n", "
\n")) - source = PollSource.query.get(source_id) - if not source: - flash('Source not found', 'error') - return redirect(url_for('admin_polling')) + @app.template_filter("timeago") + def timeago_filter(timestamp): + """Format a unix timestamp as a relative age string.""" + if not timestamp: + return "Recently" + try: + ts = float(timestamp) + except (TypeError, ValueError): + return "Recently" - source.enabled = not source.enabled - db.session.commit() - - status = 'enabled' if source.enabled else 'disabled' - flash(f'Polling {status} for {source.display_name}', 'success') - - return redirect(url_for('admin_polling')) + diff = time.time() - ts + if diff < 0: + return "Recently" + if diff < 3600: + return f"{int(diff // 60)}m ago" + if diff < 86400: + return f"{int(diff // 3600)}h ago" + if diff < 604800: + return f"{int(diff // 86400)}d ago" + return time.strftime("%Y-%m-%d", time.localtime(ts)) -@app.route('/admin/polling//update', methods=['POST']) -@login_required -def admin_polling_update(source_id): - """Update poll source configuration""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) +def register_login_loader(user_service): + """Register Flask-Login user loading.""" - from models import PollSource - - source = PollSource.query.get(source_id) - if not source: - flash('Source not found', 'error') - return redirect(url_for('admin_polling')) - - # Update all configurable fields - if request.form.get('poll_interval'): - source.poll_interval_minutes = int(request.form.get('poll_interval')) - - if request.form.get('max_posts'): - source.max_posts = int(request.form.get('max_posts')) - - if request.form.get('fetch_comments') is not None: - source.fetch_comments = request.form.get('fetch_comments') == 'true' - - if request.form.get('priority'): - source.priority = request.form.get('priority') - - if request.form.get('display_name'): - source.display_name = request.form.get('display_name') - - db.session.commit() - flash(f'Updated settings for {source.display_name}', 'success') - - return redirect(url_for('admin_polling')) + @login_manager.user_loader + def load_user(user_id): + return user_service.get_user_by_id(user_id) -@app.route('/admin/polling//poll-now', methods=['POST']) -@login_required -def admin_polling_poll_now(source_id): - """Manually trigger polling for a source""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) +def register_request_guards(app): + """Register request guards shared by page routes.""" - from models import PollSource - from polling_service import polling_service + @app.before_request + def check_first_user(): + """Redirect to admin setup if no users exist yet.""" + if request.endpoint and ( + request.endpoint.startswith("static") + or request.endpoint in ["login", "signup", "admin_setup", "serve_theme", "serve_logo"] + ): + return - source = PollSource.query.get(source_id) - if not source: - flash('Source not found', 'error') - return redirect(url_for('admin_polling')) + if current_user.is_authenticated: + return - # Trigger polling in background - try: - polling_service.poll_now(source_id) - flash(f'Polling started for {source.display_name}', 'success') - except Exception as e: - flash(f'Error starting poll: {str(e)}', 'error') - logger.error(f"Error triggering poll for {source_id}: {e}") - - return redirect(url_for('admin_polling')) + try: + if User.query.count() == 0: + return redirect(url_for("admin_setup")) + except Exception as e: + logger.warning(f"Database not ready for user count check: {e}") -@app.route('/admin/polling//delete', methods=['POST']) -@login_required -def admin_polling_delete(source_id): - """Delete a poll source""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) +def register_context_processors(app): + """Register template context processors.""" - from models import PollSource - - source = PollSource.query.get(source_id) - if not source: - flash('Source not found', 'error') - return redirect(url_for('admin_polling')) - - display_name = source.display_name - db.session.delete(source) - db.session.commit() - - flash(f'Deleted polling source: {display_name}', 'success') - logger.info(f"Admin {current_user.id} deleted poll source {source_id}") - - return redirect(url_for('admin_polling')) + @app.context_processor + def inject_app_config(): + return {"APP_NAME": app.config["APP_NAME"]} -@app.route('/admin/polling//logs') -@login_required -def admin_polling_logs(source_id): - """View logs for a specific poll source""" - if not current_user.is_admin: - flash('Access denied', 'error') - return redirect(url_for('index')) +def register_error_handlers(app): + """Register error pages for the legacy Jinja UI.""" - from models import PollSource, PollLog + @app.errorhandler(404) + def not_found(e): + return render_template("404.html"), 404 - source = PollSource.query.get(source_id) - if not source: - flash('Source not found', 'error') - return redirect(url_for('admin_polling')) - - # Get recent logs (limit to 50) - logs = source.logs.limit(50).all() - - return render_template('admin_polling_logs.html', - source=source, - logs=logs) + @app.errorhandler(500) + def server_error(e): + return render_template("500.html"), 500 -# ============================================================ -# TEMPLATE CONTEXT PROCESSORS -# ============================================================ - -@app.context_processor -def inject_app_config(): - """Inject app configuration into all templates""" - return { - 'APP_NAME': app.config['APP_NAME'] - } - - -# ============================================================ -# ERROR HANDLERS -# ============================================================ - -@app.errorhandler(404) -def not_found(e): - """404 page""" - return render_template('404.html'), 404 - - -@app.errorhandler(500) -def server_error(e): - """500 page""" - return render_template('500.html'), 500 - - -# ============================================================ -# INITIALIZATION -# ============================================================ - -if __name__ == '__main__': - print("✓ BalanceBoard starting...") - print("✓ Database: PostgreSQL with SQLAlchemy") - print("✓ Password hashing: bcrypt") - print("✓ Authentication: Flask-Login") - - app.run(host='0.0.0.0', port=DEFAULT_PORT, debug=True) +if __name__ == "__main__": + flask_app = create_app() + print("BalanceBoard starting...") + print("Database: PostgreSQL with SQLAlchemy") + print("Password hashing: bcrypt") + print("Authentication: Flask-Login") + flask_app.run(host="0.0.0.0", port=DEFAULT_PORT, debug=True) \ No newline at end of file diff --git a/blueprints/__init__.py b/blueprints/__init__.py new file mode 100644 index 0000000..6bdd842 --- /dev/null +++ b/blueprints/__init__.py @@ -0,0 +1 @@ +"""Flask blueprints for BalanceBoard.""" diff --git a/blueprints/api.py b/blueprints/api.py new file mode 100644 index 0000000..81bbcf8 --- /dev/null +++ b/blueprints/api.py @@ -0,0 +1,464 @@ +"""Versioned JSON API blueprint.""" + +import json +import logging +from datetime import datetime, timedelta + +from flask import Blueprint, jsonify, request +from flask_login import current_user, login_required + +from config import DEFAULT_PAGE_SIZE +from database import db +from extensions import get_filter_engine +from models import Bookmark +from security import is_safe_filterset +from services import get_display_name_for_source, load_platform_config, post_service + +logger = logging.getLogger(__name__) + + +FILTER_ICON_MAP = { + "no_filter": "\U0001f310", + "safe_content": "\u2705", + "tech_only": "\U0001f4bb", + "high_quality": "\u2b50", + "custom_example": "\U0001f3af", +} + + +def _current_filterset(): + """Return selected filterset from query override or current user settings.""" + filter_override = request.args.get("filter", "") + if filter_override and is_safe_filterset(filter_override): + return filter_override + + if current_user.is_authenticated: + try: + user_settings = json.loads(current_user.settings) if current_user.settings else {} + return user_settings.get("filter_set", "no_filter") + except (json.JSONDecodeError, TypeError): + pass + return "no_filter" + +FILTER_NAME_MAP = { + "no_filter": "All Content", + "safe_content": "Safe Content", + "tech_only": "Tech Only", + "high_quality": "High Quality", + "custom_example": "Custom Example", +} + + +def create_api_blueprint(name="api"): + """Create the API blueprint so it can be mounted with a version prefix.""" + bp = Blueprint(name, __name__) + + @bp.get("/posts") + def posts(): + """Get paginated posts with filtering.""" + try: + platform_config = load_platform_config() + page = int(request.args.get("page", 1)) + per_page = int(request.args.get("per_page", DEFAULT_PAGE_SIZE)) + community = request.args.get("community", "") + platform = request.args.get("platform", "") + search_query = request.args.get("q", "").lower().strip() + filter_override = request.args.get("filter", "") + + filterset_name = "no_filter" + user_communities = [] + time_filter_enabled = False + time_filter_days = 7 + if current_user.is_authenticated: + try: + user_settings = json.loads(current_user.settings) if current_user.settings else {} + filterset_name = user_settings.get("filter_set", "no_filter") + user_communities = user_settings.get("communities", []) + experience_settings = user_settings.get("experience", {}) + time_filter_enabled = experience_settings.get("time_filter_enabled", False) + time_filter_days = experience_settings.get("time_filter_days", 7) + except (json.JSONDecodeError, TypeError): + filterset_name = "no_filter" + user_communities = [] + time_filter_enabled = False + time_filter_days = 7 + + if filter_override and is_safe_filterset(filter_override): + filterset_name = filter_override + + cached_posts, cached_comments = post_service.load() + + time_cutoff = None + if time_filter_enabled: + cutoff_date = datetime.utcnow() - timedelta(days=time_filter_days) + time_cutoff = cutoff_date.timestamp() + + raw_posts = [] + for post_data in cached_posts.values(): + if time_filter_enabled and time_cutoff: + post_timestamp = post_data.get("timestamp", 0) + if post_timestamp < time_cutoff: + continue + + if community and post_data.get("source", "").lower() != community.lower(): + continue + + if platform and post_data.get("platform", "").lower() != platform.lower(): + continue + + if user_communities: + post_source = post_data.get("source", "").lower() + post_platform = post_data.get("platform", "").lower() + post_id = post_data.get("id", "").lower() + matches_community = any( + post_source == selected.lower() + or post_platform == selected.lower() + or selected.lower() in post_source + or selected.lower() in post_id + for selected in user_communities + if isinstance(selected, str) + ) + if not matches_community: + continue + + if search_query: + title = post_data.get("title", "").lower() + content = post_data.get("content", "").lower() + author = post_data.get("author", "").lower() + source = post_data.get("source", "").lower() + if not ( + search_query in title + or search_query in content + or search_query in author + or search_query in source + ): + continue + + raw_posts.append(post_data) + + filtered_posts = get_filter_engine().apply_filterset( + raw_posts, filterset_name, use_cache=True + ) + + response_posts = [] + for post_data in filtered_posts: + post_uuid = post_data.get("uuid") + source_display = get_display_name_for_source( + post_data.get("platform", ""), + post_data.get("source", ""), + platform_config, + ) + response_posts.append( + { + "id": post_uuid, + "title": post_data.get("title", "Untitled"), + "author": post_data.get("author", "Unknown"), + "platform": post_data.get("platform", "unknown"), + "score": post_data.get("score", 0), + "timestamp": post_data.get("timestamp", 0), + "url": f"/post/{post_uuid}", + "comments_count": len(cached_comments.get(post_uuid, [])), + "content_preview": (post_data.get("content", "") or "")[:200] + "..." if post_data.get("content") else "", + "source": post_data.get("source", ""), + "source_display": source_display, + "tags": post_data.get("tags", []), + "external_url": post_data.get("url", ""), + "filter_score": post_data.get("_filter_score", 0.5), + "filter_categories": post_data.get("_filter_categories", []), + "filter_tags": post_data.get("_filter_tags", []), + } + ) + + response_posts.sort(key=lambda x: (x["filter_score"], x["timestamp"]), reverse=True) + total_posts = len(response_posts) + start_idx = (page - 1) * per_page + end_idx = start_idx + per_page + total_pages = (total_posts + per_page - 1) // per_page + + return jsonify( + { + "posts": response_posts[start_idx:end_idx], + "pagination": { + "current_page": page, + "total_pages": total_pages, + "total_posts": total_posts, + "per_page": per_page, + "has_next": page < total_pages, + "has_prev": page > 1, + }, + } + ) + except Exception as e: + logger.error(f"Error loading posts: {e}") + return jsonify( + { + "posts": [], + "error": str(e), + "pagination": { + "current_page": 1, + "total_pages": 0, + "total_posts": 0, + "per_page": DEFAULT_PAGE_SIZE, + "has_next": False, + "has_prev": False, + }, + } + ) + + @bp.get("/posts/") + def post_detail(post_uuid): + """Get one post with its comment tree.""" + try: + platform_config = load_platform_config() + cached_posts, cached_comments = post_service.load() + post_data = cached_posts.get(post_uuid) + if not post_data: + return jsonify({"error": "Post not found"}), 404 + + post = dict(post_data) + post["source_display"] = get_display_name_for_source( + post.get("platform", ""), post.get("source", ""), platform_config + ) + filterset_name = _current_filterset() + filtered_comments = get_filter_engine().filter_comments( + cached_comments.get(post_uuid, []), filterset_name + ) + comments = post_service.build_comment_tree(filtered_comments) + return jsonify({"post": post, "comments": comments}) + except Exception as e: + logger.error(f"Error loading post {post_uuid}: {e}") + return jsonify({"error": "Failed to load post"}), 500 + + @bp.get("/comments/") + def comments(post_uuid): + """Get comments for a post as a tree.""" + try: + _, cached_comments = post_service.load() + filterset_name = _current_filterset() + filtered_comments = get_filter_engine().filter_comments( + cached_comments.get(post_uuid, []), filterset_name + ) + return jsonify({"comments": post_service.build_comment_tree(filtered_comments)}) + except Exception as e: + logger.error(f"Error loading comments for {post_uuid}: {e}") + return jsonify({"error": "Failed to load comments"}), 500 + + @bp.get("/platforms") + def platforms(): + """Get platform configuration and available communities.""" + try: + platform_config = load_platform_config() + communities = [] + for key, count in post_service.source_counts().items(): + platform, source = key.split(":", 1) + platform_info = platform_config.get("platforms", {}).get(platform, {}) + community_info = None + if platform_info.get("supports_communities"): + for community in platform_info.get("communities", []): + if community["id"] == source: + community_info = community + break + + if community_info: + communities.append( + { + "platform": platform, + "id": source, + "name": community_info["name"], + "display_name": community_info["display_name"], + "icon": community_info.get("icon", platform_info.get("icon", "\U0001f4c4")), + "count": count, + "description": community_info.get("description", ""), + } + ) + else: + display_name = get_display_name_for_source(platform, source, platform_config) + communities.append( + { + "platform": platform, + "id": source, + "name": source or platform, + "display_name": display_name, + "icon": platform_info.get("icon", "\U0001f4c4"), + "count": count, + "description": f"Posts from {display_name}", + } + ) + + communities.sort(key=lambda x: x["count"], reverse=True) + return jsonify( + { + "platforms": platform_config.get("platforms", {}), + "communities": communities, + "total_communities": len(communities), + } + ) + except Exception as e: + logger.error(f"Error loading platform configuration: {e}") + return jsonify({"platforms": {}, "communities": [], "total_communities": 0, "error": str(e)}) + + @bp.get("/content-timestamp") + def content_timestamp(): + """Get the last content update timestamp for auto-refresh.""" + try: + return jsonify({"timestamp": post_service.latest_content_mtime()}) + except Exception as e: + logger.error(f"Error getting content timestamp: {e}") + return jsonify({"error": "Failed to get content timestamp"}), 500 + + @bp.post("/bookmark") + @login_required + def bookmark(): + """Toggle bookmark status for a post.""" + try: + data = request.get_json() + if not data or "post_uuid" not in data: + return jsonify({"error": "Missing post_uuid"}), 400 + + post_uuid = data["post_uuid"] + if not post_uuid: + return jsonify({"error": "Invalid post_uuid"}), 400 + + existing_bookmark = Bookmark.query.filter_by( + user_id=current_user.id, post_uuid=post_uuid + ).first() + if existing_bookmark: + db.session.delete(existing_bookmark) + db.session.commit() + return jsonify({"bookmarked": False, "message": "Bookmark removed"}) + + cached_posts, _ = post_service.load() + post_data = cached_posts.get(post_uuid, {}) + new_bookmark = Bookmark( + user_id=current_user.id, + post_uuid=post_uuid, + title=post_data.get("title", ""), + platform=post_data.get("platform", ""), + source=post_data.get("source", ""), + ) + db.session.add(new_bookmark) + db.session.commit() + return jsonify({"bookmarked": True, "message": "Bookmark added"}) + except Exception as e: + db.session.rollback() + logger.error(f"Error toggling bookmark: {e}") + return jsonify({"error": "Failed to toggle bookmark"}), 500 + + @bp.get("/bookmarks") + @login_required + def bookmarks(): + """Get the current user's bookmarks.""" + try: + page = int(request.args.get("page", 1)) + per_page = int(request.args.get("per_page", DEFAULT_PAGE_SIZE)) + bookmarks_query = Bookmark.query.filter_by(user_id=current_user.id).order_by( + Bookmark.created_at.desc() + ) + total_bookmarks = bookmarks_query.count() + bookmark_rows = bookmarks_query.offset((page - 1) * per_page).limit(per_page).all() + cached_posts, cached_comments = post_service.load() + + bookmark_posts = [] + for bookmark_row in bookmark_rows: + post_data = cached_posts.get(bookmark_row.post_uuid) + if post_data: + bookmark_posts.append( + { + "id": bookmark_row.post_uuid, + "title": post_data.get("title", bookmark_row.title or "Untitled"), + "author": post_data.get("author", "Unknown"), + "platform": post_data.get("platform", bookmark_row.platform or "unknown"), + "score": post_data.get("score", 0), + "timestamp": post_data.get("timestamp", 0), + "url": f"/post/{bookmark_row.post_uuid}", + "comments_count": len(cached_comments.get(bookmark_row.post_uuid, [])), + "content_preview": (post_data.get("content", "") or "")[:200] + "..." if post_data.get("content") else "", + "source": post_data.get("source", bookmark_row.source or ""), + "bookmarked_at": bookmark_row.created_at.isoformat(), + "external_url": post_data.get("url", ""), + } + ) + else: + bookmark_posts.append( + { + "id": bookmark_row.post_uuid, + "title": bookmark_row.title or "Untitled", + "author": "Unknown", + "platform": bookmark_row.platform or "unknown", + "score": 0, + "timestamp": 0, + "url": f"/post/{bookmark_row.post_uuid}", + "comments_count": 0, + "content_preview": "Content no longer available", + "source": bookmark_row.source or "", + "bookmarked_at": bookmark_row.created_at.isoformat(), + "external_url": "", + "archived": True, + } + ) + + total_pages = (total_bookmarks + per_page - 1) // per_page + return jsonify( + { + "posts": bookmark_posts, + "pagination": { + "current_page": page, + "total_pages": total_pages, + "total_posts": total_bookmarks, + "per_page": per_page, + "has_next": page < total_pages, + "has_prev": page > 1, + }, + } + ) + except Exception as e: + logger.error(f"Error getting bookmarks: {e}") + return jsonify({"error": "Failed to get bookmarks"}), 500 + + @bp.get("/bookmark-status/") + @login_required + def bookmark_status(post_uuid): + """Check if a post is bookmarked by the current user.""" + try: + bookmark_row = Bookmark.query.filter_by( + user_id=current_user.id, post_uuid=post_uuid + ).first() + return jsonify({"bookmarked": bookmark_row is not None}) + except Exception as e: + logger.error(f"Error checking bookmark status: {e}") + return jsonify({"error": "Failed to check bookmark status"}), 500 + + @bp.get("/filters") + def filters(): + """Get available filtersets.""" + try: + filter_rows = [] + current_filter = "no_filter" + if current_user.is_authenticated: + try: + user_settings = json.loads(current_user.settings) if current_user.settings else {} + current_filter = user_settings.get("filter_set", "no_filter") + except (json.JSONDecodeError, TypeError): + pass + + filter_engine = get_filter_engine() + for filterset_name in filter_engine.get_available_filtersets(): + filterset_config = filter_engine.config.get_filterset(filterset_name) + if filterset_config: + filter_rows.append( + { + "id": filterset_name, + "name": FILTER_NAME_MAP.get( + filterset_name, filterset_name.replace("_", " ").title() + ), + "description": filterset_config.get("description", ""), + "icon": FILTER_ICON_MAP.get(filterset_name, "\U0001f527"), + "active": filterset_name == current_filter, + } + ) + return jsonify({"filters": filter_rows}) + except Exception as e: + logger.error(f"Error getting filters: {e}") + return jsonify({"error": "Failed to get filters"}), 500 + + return bp \ No newline at end of file diff --git a/comment_lib.py b/comment_lib.py deleted file mode 100644 index 118694e..0000000 --- a/comment_lib.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -Comment Library -Atomic functions for comment processing and tree manipulation. -""" - -import json -from pathlib import Path -from typing import List, Dict, Optional - - -class comment_lib: - """Atomic comment processing functions""" - - @staticmethod - def build_comment_tree(flat_comments: List[Dict]) -> List[Dict]: - """ - Convert flat array of comments to nested tree structure. - Returns list of root-level comments with nested children. - """ - if not flat_comments: - return [] - - # Create lookup dict - comment_map = {c['uuid']: {**c, 'children': []} for c in flat_comments} - - # Build tree - roots = [] - for comment in flat_comments: - parent_uuid = comment.get('parent_comment_uuid') - if parent_uuid and parent_uuid in comment_map: - comment_map[parent_uuid]['children'].append(comment_map[comment['uuid']]) - else: - roots.append(comment_map[comment['uuid']]) - - return roots - - @staticmethod - def flatten_comment_tree(tree: List[Dict]) -> List[Dict]: - """ - Convert nested tree structure to flat array. - Removes 'children' key from each comment. - """ - flat = [] - - def traverse(nodes): - for node in nodes: - children = node.pop('children', []) - flat.append(node) - if children: - traverse(children) - - traverse(tree) - return flat - - @staticmethod - def load_comments_for_post(post_uuid: str, data_dir: str) -> List[Dict]: - """ - Load all comment files linked to a post. - Scans comment directory for comments with matching post_uuid. - """ - comments_dir = Path(data_dir) / 'comments' - if not comments_dir.exists(): - return [] - - comments = [] - for comment_file in comments_dir.glob('*.json'): - with open(comment_file, 'r') as f: - comment = json.load(f) - if comment.get('post_uuid') == post_uuid: - comments.append(comment) - - return comments - - @staticmethod - def sort_comments(comments: List[Dict], by: str = 'score', order: str = 'desc') -> List[Dict]: - """ - Sort comments by specified field. - - Args: - comments: List of comment dicts - by: Field to sort by ('score', 'timestamp', 'depth', 'author') - order: 'asc' or 'desc' - - Returns: - Sorted list of comments - """ - reverse = (order == 'desc') - - return sorted(comments, key=lambda c: c.get(by, 0), reverse=reverse) - - @staticmethod - def get_comment_depth(comment: Dict, comment_map: Dict) -> int: - """ - Calculate actual depth of a comment by traversing up parent chain. - Useful for recalculating depth after filtering. - """ - depth = 0 - current_uuid = comment.get('parent_comment_uuid') - - while current_uuid and current_uuid in comment_map: - depth += 1 - current_uuid = comment_map[current_uuid].get('parent_comment_uuid') - - return depth - - @staticmethod - def get_comment_stats(comments: List[Dict]) -> Dict: - """ - Get statistics about a comment list. - - Returns: - Dict with total, max_depth, avg_score, etc. - """ - if not comments: - return { - 'total': 0, - 'max_depth': 0, - 'avg_score': 0, - 'total_score': 0 - } - - depths = [c.get('depth', 0) for c in comments] - scores = [c.get('score', 0) for c in comments] - - return { - 'total': len(comments), - 'max_depth': max(depths) if depths else 0, - 'avg_score': sum(scores) / len(scores) if scores else 0, - 'total_score': sum(scores) - } - - @staticmethod - def filter_by_depth(comments: List[Dict], max_depth: int) -> List[Dict]: - """ - Filter comments to only include those at or below max_depth. - """ - return [c for c in comments if c.get('depth', 0) <= max_depth] - - @staticmethod - def get_top_level_comments(comments: List[Dict]) -> List[Dict]: - """ - Get only top-level comments (depth 0, no parent). - """ - return [c for c in comments if c.get('depth', 0) == 0 or not c.get('parent_comment_uuid')] - - @staticmethod - def count_replies(comment_uuid: str, comments: List[Dict]) -> int: - """ - Count total number of replies (direct and nested) for a comment. - """ - count = 0 - - for comment in comments: - if comment.get('parent_comment_uuid') == comment_uuid: - count += 1 - # Recursively count this comment's replies - count += comment_lib.count_replies(comment['uuid'], comments) - - return count diff --git a/config.py b/config.py new file mode 100644 index 0000000..f4d9009 --- /dev/null +++ b/config.py @@ -0,0 +1,43 @@ +"""Application configuration. + +Centralizes environment-driven Flask config and app-wide constants so they +can be imported by the app factory, services, and route modules without each +reaching into ``os.getenv`` independently. +""" + +import os + +from dotenv import load_dotenv + +# Load .env once when the config module is imported. +load_dotenv() + + +class Config: + """Flask configuration loaded from the environment.""" + + SECRET_KEY = os.getenv("SECRET_KEY", "dev-secret-key-change-in-production") + MAX_CONTENT_LENGTH = 16 * 1024 * 1024 # 16MB max request body (avatar uploads) + ALLOW_ANONYMOUS_ACCESS = os.getenv("ALLOW_ANONYMOUS_ACCESS", "true").lower() == "true" + + # Branding + APP_NAME = os.getenv("APP_NAME", "BalanceBoard") + LOGO_PATH = os.getenv("LOGO_PATH", "logo.png") + + # Auth0 (optional; empty disables OAuth login) + AUTH0_DOMAIN = os.getenv("AUTH0_DOMAIN", "") + AUTH0_CLIENT_ID = os.getenv("AUTH0_CLIENT_ID", "") + AUTH0_CLIENT_SECRET = os.getenv("AUTH0_CLIENT_SECRET", "") + AUTH0_AUDIENCE = os.getenv("AUTH0_AUDIENCE", "") + + +# App-wide constants (not Flask config keys). Imported directly by modules. +ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg", "gif"} +UPLOAD_FOLDER = "static/avatars" +MAX_FILENAME_LENGTH = 100 +DEFAULT_PORT = 5021 +DEFAULT_PAGE_SIZE = 20 +MIN_PASSWORD_LENGTH = 8 +MAX_USERNAME_LENGTH = 80 +MAX_EMAIL_LENGTH = 120 +MAX_COMMUNITY_NAME_LENGTH = 100 \ No newline at end of file diff --git a/data_collection.py b/data_collection.py index 674630d..840b2c7 100644 --- a/data_collection.py +++ b/data_collection.py @@ -11,6 +11,8 @@ from datetime import datetime, timedelta from pathlib import Path from typing import List, Dict, Tuple from data_collection_lib import data_methods +from database import db +from models import Comment, Post # ===== STORAGE FUNCTIONS ===== @@ -103,6 +105,52 @@ def create_moderation_stub(target_id: str, target_type: str, dirs: Dict) -> str: return mod_uuid + + +def upsert_post_record(post: Dict): + """Best-effort DB upsert; JSON files remain an archive/export artifact.""" + try: + db.session.merge(Post( + uuid=post["uuid"], + external_id=post.get("id"), + platform=post.get("platform", "") or "", + source=post.get("source", "") or "", + title=(post.get("title") or "")[:500], + author=post.get("author"), + url=post.get("url"), + content=post.get("content"), + score=int(post.get("score", 0) or 0), + timestamp=int(post.get("timestamp", 0) or 0), + tags=post.get("tags"), + moderation_uuid=post.get("moderation_uuid"), + )) + db.session.commit() + except Exception as e: + db.session.rollback() + print(f"Warning: could not persist post {post.get('uuid')} to DB: {e}") + + +def upsert_comment_record(comment: Dict): + """Best-effort DB upsert for collected comments.""" + try: + db.session.merge(Comment( + uuid=comment["uuid"], + post_uuid=comment.get("post_uuid") or "", + platform=comment.get("platform"), + parent_comment_uuid=comment.get("parent_comment_uuid"), + comment_id=comment.get("comment_id") or comment.get("id"), + author=comment.get("author"), + content=comment.get("content"), + score=int(comment.get("score", 0) or 0), + timestamp=int(comment.get("timestamp", 0) or 0), + depth=int(comment.get("depth", 0) or 0), + moderation_uuid=comment.get("moderation_uuid"), + )) + db.session.commit() + except Exception as e: + db.session.rollback() + print(f"Warning: could not persist comment {comment.get('uuid')} to DB: {e}") + # ===== POST FUNCTIONS ===== def save_post(post: Dict, platform: str, index: Dict, dirs: Dict) -> str: @@ -122,6 +170,8 @@ def save_post(post: Dict, platform: str, index: Dict, dirs: Dict) -> str: with open(post_file, 'w') as f: json.dump(post, f, indent=2) + upsert_post_record(post) + # Update index index[post_id] = post_uuid @@ -147,6 +197,8 @@ def save_comment(comment: Dict, post_uuid: str, platform: str, dirs: Dict) -> st with open(comment_file, 'w') as f: json.dump(comment, f, indent=2) + upsert_comment_record(comment) + return comment_uuid diff --git a/data_collection_lib.py b/data_collection_lib.py index dd30194..c8cd988 100644 --- a/data_collection_lib.py +++ b/data_collection_lib.py @@ -2,6 +2,7 @@ import requests import json import datetime as dt import time +from platforms import discover_modules, get_platform_fetcher from collections import defaultdict, deque @@ -70,21 +71,12 @@ _rate_limiter = RateLimiter() class data_methods(): @staticmethod def getData(platform, start_date, end_date, community, max_posts): - if platform == "reddit": - return data_methods.fetchers.getRedditData(start_date, end_date, community, max_posts) - elif platform == "pushshift": - return data_methods.fetchers.getPushshiftData(start_date, end_date, community, max_posts) - elif platform == "hackernews": - return data_methods.fetchers.getHackerNewsData(start_date, end_date, community, max_posts) - elif platform == "lobsters": - return data_methods.fetchers.getLobstersData(start_date, end_date, community, max_posts) - elif platform == "stackexchange": - return data_methods.fetchers.getStackExchangeData(start_date, end_date, community, max_posts) - elif platform == "rss": - return data_methods.fetchers.getRSSData(start_date, end_date, community, max_posts) - else: + discover_modules(["platforms.builtins"]) + fetcher = get_platform_fetcher(platform) + if not fetcher: print("dataGrab.getData: platform not recognized") return None + return fetcher.fetch_posts(start_date, end_date, community, max_posts) # ===== ATOMIC UTILITY FUNCTIONS ===== class utils(): diff --git a/docker-compose.yml b/docker-compose.yml index 44c21d2..e8285b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,6 @@ services: - ./data:/app/data - ./static:/app/static - ./backups:/app/backups - - ./active_html:/app/active_html depends_on: postgres: condition: service_healthy diff --git a/extensions.py b/extensions.py new file mode 100644 index 0000000..946851b --- /dev/null +++ b/extensions.py @@ -0,0 +1,43 @@ +"""Shared extension instances and accessors. + +Extensions are created here (unbound to any app) and initialized against the +app inside ``create_app()``. This lets blueprints/services import the single +shared ``login_manager`` / ``oauth`` without importing ``app`` itself. + +``filter_engine`` and ``polling_service`` are accessed lazily so importing this +module (or ``app``) has no side effects. +""" + +import logging + +from flask_login import LoginManager +from authlib.integrations.flask_client import OAuth + +logger = logging.getLogger(__name__) + +# Flask-Login: bound to the app in create_app(). +login_manager = LoginManager() +login_manager.login_view = "login" +login_manager.login_message = "Please log in to access this page." + +# Authlib OAuth: bound to the app in create_app(). +oauth = OAuth() + + +def get_filter_engine(): + """Return the singleton FilterEngine instance (lazy). + + ``FilterEngine.get_instance()`` is itself a lazy singleton, so this is + cheap to call from routes/services. Kept as a function (not a module + global) so importing this module never triggers filter-engine init. + """ + from filter_pipeline import FilterEngine + + return FilterEngine.get_instance() + + +def get_polling_service(): + """Return the shared polling service singleton (lazy import).""" + from polling_service import polling_service + + return polling_service \ No newline at end of file diff --git a/filter_config.json b/filter_config.json index 7b19b2e..13f6bc2 100644 --- a/filter_config.json +++ b/filter_config.json @@ -16,9 +16,31 @@ "filterset_cache_ttl_hours": 24 }, "pipeline": { - "default_stages": ["categorizer", "moderator", "filter", "ranker"], + "default_stages": ["categorizer", "moderator", "filter", "plugins", "ranker"], "batch_size": 50, - "enable_parallel": true + "enable_parallel": true, + "stage_modules": [] + }, + "plugins": { + "modules": [], + "enabled": ["keyword", "quality"], + "configs": { + "keyword": { + "enabled": true, + "blocklist": [], + "allowlist": [], + "check_title": true, + "check_content": true + }, + "quality": { + "enabled": true, + "min_title_length": 10, + "max_title_length": 300, + "min_content_length": 0, + "max_caps_ratio": 0.5, + "max_exclamation_marks": 3 + } + } }, "output": { "filtered_dir": "data/filtered", diff --git a/filter_lib.py b/filter_lib.py deleted file mode 100644 index 8e2a818..0000000 --- a/filter_lib.py +++ /dev/null @@ -1,345 +0,0 @@ -""" -Filter Library -Bare bones utilities for filtering posts and comments based on rules. -""" - -import json -from pathlib import Path -from typing import Dict, List, Any, Optional -from abc import ABC, abstractmethod - - -class filter_lib: - """Atomic filter utility functions""" - - @staticmethod - def load_filterset(path: str) -> Dict: - """Load filterset JSON from file""" - with open(path, 'r') as f: - return json.load(f) - - @staticmethod - def load_data_by_uuid(uuid: str, data_dir: str) -> Optional[Dict]: - """Load single JSON file by UUID""" - file_path = Path(data_dir) / f"{uuid}.json" - if not file_path.exists(): - return None - - with open(file_path, 'r') as f: - return json.load(f) - - @staticmethod - def merge_moderation(item: Dict, moderation_data: Dict) -> Dict: - """Merge item with its moderation data by UUID""" - mod_uuid = item.get('moderation_uuid') - if mod_uuid and mod_uuid in moderation_data: - item['moderation'] = moderation_data[mod_uuid] - else: - item['moderation'] = {} - return item - - @staticmethod - def get_nested_value(obj: Dict, path: str) -> Any: - """Get value from nested dict using dot notation (e.g., 'moderation.flags.is_safe')""" - keys = path.split('.') - value = obj - for key in keys: - if isinstance(value, dict) and key in value: - value = value[key] - else: - return None - return value - - @staticmethod - def evaluate_rule(value: Any, operator: str, target: Any) -> bool: - """Evaluate single rule: value operator target""" - if value is None: - return False - - if operator == 'equals': - return value == target - elif operator == 'not_equals': - return value != target - elif operator == 'in': - return value in target - elif operator == 'not_in': - return value not in target - elif operator == 'min': - return value >= target - elif operator == 'max': - return value <= target - elif operator == 'after': - return value > target - elif operator == 'before': - return value < target - elif operator == 'contains': - return target in value - elif operator == 'excludes': - if isinstance(value, list): - return not any(item in target for item in value) - return target not in value - elif operator == 'includes': - if isinstance(value, list): - return target in value - return False - elif operator == 'includes_any': - # Special case for topic matching - if isinstance(value, list) and isinstance(target, list): - for topic_item in value: - for rule in target: - if (topic_item.get('topic') == rule.get('topic') and - topic_item.get('confidence', 0) >= rule.get('confidence_min', 0)): - return True - return False - elif operator == 'min_length': - return len(str(value)) >= target - elif operator == 'max_length': - return len(str(value)) <= target - else: - return False - - @staticmethod - def apply_rules(item: Dict, rules: Dict) -> bool: - """ - Apply multiple rules to item, return True if all pass (AND logic). - Rules format: {"field.path": {"operator": value}} - """ - if not rules: - return True # Empty rules = pass all - - for field_path, rule_def in rules.items(): - value = filter_lib.get_nested_value(item, field_path) - - # Support multiple operators per field - for operator, target in rule_def.items(): - if not filter_lib.evaluate_rule(value, operator, target): - return False - - return True - - -class CommentFilterMode(ABC): - """Abstract base class for comment filtering modes""" - - @staticmethod - @abstractmethod - def filter(comments: List[Dict], rules: Dict, moderation_data: Dict) -> List[Dict]: - """Filter comments based on rules and moderation data. Override in subclasses.""" - pass - - -class TreePruningMode(CommentFilterMode): - """ - Tree Pruning Filter Mode (Default) - Fruit of the poisonous tree: if parent fails moderation, remove all children. - """ - - @staticmethod - def filter(comments: List[Dict], rules: Dict, moderation_data: Dict) -> List[Dict]: - """ - Filter comments using tree pruning. - Build tree structure, evaluate from root down, prune toxic branches. - """ - if not comments: - return [] - - # Merge moderation data into comments - for comment in comments: - filter_lib.merge_moderation(comment, moderation_data) - - # Build tree structure - tree = TreePruningMode._build_tree(comments) - - # Prune tree based on rules - pruned = TreePruningMode._prune_tree(tree, rules) - - # Flatten back to list - return TreePruningMode._flatten_tree(pruned) - - @staticmethod - def _build_tree(comments: List[Dict]) -> List[Dict]: - """Build nested tree from flat comment list""" - # Create lookup dict - comment_map = {c['uuid']: {**c, 'children': []} for c in comments} - - # Build tree - roots = [] - for comment in comments: - parent_uuid = comment.get('parent_comment_uuid') - if parent_uuid and parent_uuid in comment_map: - comment_map[parent_uuid]['children'].append(comment_map[comment['uuid']]) - else: - roots.append(comment_map[comment['uuid']]) - - return roots - - @staticmethod - def _prune_tree(tree: List[Dict], rules: Dict) -> List[Dict]: - """ - Recursively prune tree. - If node fails rules, remove it and all children. - """ - pruned = [] - - for node in tree: - # Check if this node passes rules - if filter_lib.apply_rules(node, rules): - # Node passes, recursively check children - if node.get('children'): - node['children'] = TreePruningMode._prune_tree(node['children'], rules) - pruned.append(node) - # If node fails, it and all children are discarded (tree pruning) - - return pruned - - @staticmethod - def _flatten_tree(tree: List[Dict]) -> List[Dict]: - """Flatten tree back to list""" - flat = [] - - def traverse(nodes): - for node in nodes: - children = node.pop('children', []) - flat.append(node) - if children: - traverse(children) - - traverse(tree) - return flat - - -class IndividualFilterMode(CommentFilterMode): - """ - Individual Filter Mode - Each comment evaluated independently, no tree pruning. - """ - - @staticmethod - def filter(comments: List[Dict], rules: Dict, moderation_data: Dict) -> List[Dict]: - """Filter comments individually""" - filtered = [] - - for comment in comments: - # Merge moderation - filter_lib.merge_moderation(comment, moderation_data) - - # Apply rules - if filter_lib.apply_rules(comment, rules): - filtered.append(comment) - - return filtered - - -class ScoreBasedFilterMode(CommentFilterMode): - """ - Score-Based Filter Mode - Filter comments based on score thresholds, keeping high-quality content. - """ - - @staticmethod - def filter(comments: List[Dict], rules: Dict, moderation_data: Dict) -> List[Dict]: - """Filter comments based on score and rules""" - filtered = [] - - for comment in comments: - # Merge moderation - filter_lib.merge_moderation(comment, moderation_data) - - # Apply basic rules first - if not filter_lib.apply_rules(comment, rules): - continue - - # Additional score-based filtering - score = comment.get('score', 0) - min_score = rules.get('score', {}).get('min', -1000) # Default very low threshold - - if score >= min_score: - filtered.append(comment) - - return filtered - - -class TimeBoundFilterMode(CommentFilterMode): - """ - Time-Bound Filter Mode - Filter comments within specific time ranges. - """ - - @staticmethod - def filter(comments: List[Dict], rules: Dict, moderation_data: Dict) -> List[Dict]: - """Filter comments within time bounds""" - from datetime import datetime - - filtered = [] - - for comment in comments: - # Merge moderation - filter_lib.merge_moderation(comment, moderation_data) - - # Apply basic rules first - if not filter_lib.apply_rules(comment, rules): - continue - - # Time-based filtering - timestamp = comment.get('timestamp') - if timestamp: - try: - comment_time = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) - time_rules = rules.get('timestamp', {}) - - after = time_rules.get('after') - before = time_rules.get('before') - - if after: - after_time = datetime.fromisoformat(after.replace('Z', '+00:00')) - if comment_time <= after_time: - continue - - if before: - before_time = datetime.fromisoformat(before.replace('Z', '+00:00')) - if comment_time >= before_time: - continue - - filtered.append(comment) - except (ValueError, TypeError): - # Skip malformed timestamps - continue - else: - # No timestamp, include if no time rules - if 'timestamp' not in rules: - filtered.append(comment) - - return filtered - - -class ContentLengthFilterMode(CommentFilterMode): - """ - Content Length Filter Mode - Filter comments based on content length criteria. - """ - - @staticmethod - def filter(comments: List[Dict], rules: Dict, moderation_data: Dict) -> List[Dict]: - """Filter comments based on content length""" - filtered = [] - - for comment in comments: - # Merge moderation - filter_lib.merge_moderation(comment, moderation_data) - - # Apply basic rules first - if not filter_lib.apply_rules(comment, rules): - continue - - # Content length filtering - content = comment.get('content', '') - content_length = len(content) - - length_rules = rules.get('content_length', {}) - min_length = length_rules.get('min', 0) - max_length = length_rules.get('max', float('inf')) - - if min_length <= content_length <= max_length: - filtered.append(comment) - - return filtered diff --git a/filter_pipeline/__init__.py b/filter_pipeline/__init__.py index 8e88f62..de45513 100644 --- a/filter_pipeline/__init__.py +++ b/filter_pipeline/__init__.py @@ -5,6 +5,13 @@ Content filtering, categorization, and ranking system for BalanceBoard. from .engine import FilterEngine from .models import FilterResult, ProcessingStatus +from .registry import register_stage, register_plugin -__all__ = ['FilterEngine', 'FilterResult', 'ProcessingStatus'] -__version__ = '1.0.0' +__all__ = [ + 'FilterEngine', + 'FilterResult', + 'ProcessingStatus', + 'register_stage', + 'register_plugin', +] +__version__ = '1.0.0' \ No newline at end of file diff --git a/filter_pipeline/engine.py b/filter_pipeline/engine.py index d6220e9..22fd2cc 100644 --- a/filter_pipeline/engine.py +++ b/filter_pipeline/engine.py @@ -13,6 +13,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from .config import FilterConfig from .cache import FilterCache from .models import FilterResult, ProcessingStatus, AIAnalysisResult +from .registry import discover_modules, get_registered_stages logger = logging.getLogger(__name__) @@ -56,24 +57,36 @@ class FilterEngine: return cls._instance def _init_stages(self): - """Initialize pipeline stages (lazy loading)""" + """Initialize pipeline stages from the registry (lazy loading).""" if self._stages is not None: return - from .stages.categorizer import CategorizerStage - from .stages.moderator import ModeratorStage - from .stages.filter import FilterStage - from .stages.ranker import RankerStage + # Import built-ins and any configured extension modules for decorator + # side effects. This keeps engine orchestration independent of concrete + # stage classes and gives plugins a zero-core-edit registration path. + discover_modules([ + 'filter_pipeline.stages.categorizer', + 'filter_pipeline.stages.moderator', + 'filter_pipeline.stages.filter', + 'filter_pipeline.stages.ranker', + 'filter_pipeline.stages.plugins', + 'filter_pipeline.stages.comment_filter', + 'filter_pipeline.plugins.keyword', + 'filter_pipeline.plugins.quality', + *self.config.config.get('pipeline', {}).get('stage_modules', []), + *self.config.config.get('plugins', {}).get('modules', []), + ]) - # Initialize stages based on configuration self._stages = { - 'categorizer': CategorizerStage(self.config, self.cache), - 'moderator': ModeratorStage(self.config, self.cache), - 'filter': FilterStage(self.config, self.cache), - 'ranker': RankerStage(self.config, self.cache) + name: stage_cls(self.config, self.cache) + for name, stage_cls in get_registered_stages().items() } - logger.info(f"Initialized {len(self._stages)} pipeline stages") + logger.info( + "Initialized %s registered pipeline stages: %s", + len(self._stages), + ', '.join(sorted(self._stages.keys())) + ) def apply_filterset( self, @@ -180,24 +193,26 @@ class FilterEngine: if filterset_name == 'no_filter': return self._process_no_filter(posts) - # Initialize stages if needed - if self.config.is_ai_enabled(): - self._init_stages() - - # If AI is disabled but the filterset requires it, do NOT silently pass - # everything as no_filter. Pass the posts through (so the feed is not - # blanked) but mark every result as FAILED with an explicit error so the - # degradation is observable, not silent. - if not self.config.is_ai_enabled() and filterset_name != 'no_filter': - logger.warning( - f"AI disabled but filterset '{filterset_name}' requires AI - " - f"passing posts through unfiltered with FAILED status" - ) - return self._process_ai_disabled(filterset_name, posts) + # Initialize stages (registry-driven). This must happen regardless of + # whether AI is enabled: offline filtersets (rules/plugins/ranker) still + # need their stages instantiated to run. + self._init_stages() # Get pipeline stages for this filterset stage_names = self._get_stages_for_filterset(filterset_name) + # If AI is disabled but the filterset's stages require AI, do NOT silently + # pass everything as no_filter. Pass the posts through (so the feed is not + # blanked) but mark every result as FAILED with an explicit error so the + # degradation is observable, not silent. Filtersets whose stages are all + # offline (filter/plugins/ranker/comment_filter) still run normally. + if not self.config.is_ai_enabled() and self._stages_need_ai(stage_names): + logger.warning( + f"AI disabled but filterset '{filterset_name}' requires AI stages " + f"({stage_names}) - passing posts through unfiltered with FAILED status" + ) + return self._process_ai_disabled(filterset_name, posts) + # Process posts (parallel or sequential based on config) if self.config.is_parallel_enabled(): results = self._process_batch_parallel(posts, filterset_name, stage_names) @@ -206,6 +221,16 @@ class FilterEngine: return results + def _stages_need_ai(self, stage_names: List[str]) -> bool: + """Return True if any named stage class declares ``requires_ai``.""" + from .registry import get_stage_class + + for name in stage_names: + stage_cls = get_stage_class(name) + if stage_cls is not None and getattr(stage_cls, 'requires_ai', False): + return True + return False + def _process_no_filter(self, posts: List[Dict[str, Any]]) -> List[FilterResult]: """Process posts with no_filter (all pass with default scores)""" results = [] @@ -402,3 +427,27 @@ class FilterEngine: self.config.reload() self._stages = None # Force re-initialization of stages logger.info("Configuration reloaded") + + def filter_comments( + self, + comments: List[Dict[str, Any]], + filterset_name: str = 'no_filter' + ) -> List[Dict[str, Any]]: + """Filter a post's flat comment list according to a filterset. + + Comment filtering is tree-shaped (per post) and lives in the registered + ``comment_filter`` stage rather than the per-post stage pipeline. The + caller (API endpoint) builds the tree from the returned flat list via + ``PostService.build_comment_tree``. + + Fails open: if the ``comment_filter`` stage is not registered, the + comments are returned unchanged so a missing stage never blanks them. + """ + if not comments: + return [] + self._init_stages() + comment_stage = self._stages.get('comment_filter') + if not comment_stage: + logger.warning("comment_filter stage not registered; returning comments unfiltered") + return comments + return comment_stage.filter_comments(comments, filterset_name) diff --git a/filter_pipeline/plugins/keyword.py b/filter_pipeline/plugins/keyword.py index ffc61d6..951bf09 100644 --- a/filter_pipeline/plugins/keyword.py +++ b/filter_pipeline/plugins/keyword.py @@ -7,10 +7,12 @@ import logging from typing import Dict, Any, Optional, List from .base import BaseFilterPlugin +from ..registry import register_plugin logger = logging.getLogger(__name__) +@register_plugin("keyword") class KeywordFilterPlugin(BaseFilterPlugin): """ Filter posts based on keyword matching. diff --git a/filter_pipeline/plugins/quality.py b/filter_pipeline/plugins/quality.py index 065f6cb..3f5fa47 100644 --- a/filter_pipeline/plugins/quality.py +++ b/filter_pipeline/plugins/quality.py @@ -8,10 +8,12 @@ import re from typing import Dict, Any, Optional from .base import BaseFilterPlugin +from ..registry import register_plugin logger = logging.getLogger(__name__) +@register_plugin("quality") class QualityFilterPlugin(BaseFilterPlugin): """ Filter posts based on quality metrics. diff --git a/filter_pipeline/registry.py b/filter_pipeline/registry.py new file mode 100644 index 0000000..06b708a --- /dev/null +++ b/filter_pipeline/registry.py @@ -0,0 +1,62 @@ +"""Registries for filter pipeline stages and plugins.""" + +import importlib +import logging +from typing import Any, Dict, Iterable, Optional, Type + +logger = logging.getLogger(__name__) + +_STAGE_REGISTRY: Dict[str, Type[Any]] = {} +_PLUGIN_REGISTRY: Dict[str, Type[Any]] = {} +_DISCOVERED_MODULES = set() + + +def register_stage(name: str): + """Register a pipeline stage class under a config name.""" + def decorator(stage_cls: Type[Any]): + if name in _STAGE_REGISTRY and _STAGE_REGISTRY[name] is not stage_cls: + logger.warning("Replacing registered filter stage '%s'", name) + _STAGE_REGISTRY[name] = stage_cls + return stage_cls + + return decorator + + +def register_plugin(name: str): + """Register a filter plugin class under a config name.""" + def decorator(plugin_cls: Type[Any]): + if name in _PLUGIN_REGISTRY and _PLUGIN_REGISTRY[name] is not plugin_cls: + logger.warning("Replacing registered filter plugin '%s'", name) + _PLUGIN_REGISTRY[name] = plugin_cls + return plugin_cls + + return decorator + + +def get_stage_class(name: str) -> Optional[Type[Any]]: + """Return a registered stage class by name.""" + return _STAGE_REGISTRY.get(name) + + +def get_plugin_class(name: str) -> Optional[Type[Any]]: + """Return a registered plugin class by name.""" + return _PLUGIN_REGISTRY.get(name) + + +def get_registered_stages() -> Dict[str, Type[Any]]: + """Return a copy of registered stage classes.""" + return dict(_STAGE_REGISTRY) + + +def get_registered_plugins() -> Dict[str, Type[Any]]: + """Return a copy of registered plugin classes.""" + return dict(_PLUGIN_REGISTRY) + + +def discover_modules(module_names: Iterable[str]): + """Import modules for registration side effects once.""" + for module_name in module_names: + if not module_name or module_name in _DISCOVERED_MODULES: + continue + importlib.import_module(module_name) + _DISCOVERED_MODULES.add(module_name) \ No newline at end of file diff --git a/filter_pipeline/rules.py b/filter_pipeline/rules.py new file mode 100644 index 0000000..8e5f1d2 --- /dev/null +++ b/filter_pipeline/rules.py @@ -0,0 +1,80 @@ +"""Shared rule evaluation for posts and comments.""" + +from typing import Any, Dict + + +def get_nested_value(obj: Dict[str, Any], path: str) -> Any: + """Get a nested dict value using dot notation.""" + value = obj + for key in path.split("."): + if isinstance(value, dict) and key in value: + value = value[key] + else: + return None + return value + + +def evaluate_rule(value: Any, operator: str, target: Any) -> bool: + """Evaluate one rule operator.""" + if value is None: + return False + + if operator == "equals": + return value == target + if operator == "not_equals": + return value != target + if operator == "in": + return value in target + if operator == "not_in": + return value not in target + if operator == "min": + return value >= target + if operator == "max": + return value <= target + if operator == "after": + return value > target + if operator == "before": + return value < target + if operator == "contains": + return target in value + if operator == "excludes": + if isinstance(value, list): + return not any(item in target for item in value) + return value not in target + if operator == "includes": + if isinstance(value, list): + return target in value + return False + if operator == "includes_any": + if isinstance(value, list) and isinstance(target, list): + for item in value: + if isinstance(item, dict): + for rule in target: + if ( + isinstance(rule, dict) + and item.get("topic") == rule.get("topic") + and item.get("confidence", 0) >= rule.get("confidence_min", 0) + ): + return True + elif item in target: + return True + return False + if operator == "min_length": + return len(str(value)) >= target + if operator == "max_length": + return len(str(value)) <= target + + return False + + +def apply_rules(item: Dict[str, Any], rules: Dict[str, Dict[str, Any]]) -> bool: + """Return True when all field rules pass.""" + if not rules: + return True + + for field_path, rule_def in rules.items(): + value = get_nested_value(item, field_path) + for operator, target in rule_def.items(): + if not evaluate_rule(value, operator, target): + return False + return True \ No newline at end of file diff --git a/filter_pipeline/stages/__init__.py b/filter_pipeline/stages/__init__.py index 4201b06..31377c6 100644 --- a/filter_pipeline/stages/__init__.py +++ b/filter_pipeline/stages/__init__.py @@ -8,5 +8,15 @@ from .categorizer import CategorizerStage from .moderator import ModeratorStage from .filter import FilterStage from .ranker import RankerStage +from .plugins import PluginStage +from .comment_filter import CommentFilterStage -__all__ = ['BaseStage', 'CategorizerStage', 'ModeratorStage', 'FilterStage', 'RankerStage'] +__all__ = [ + 'BaseStage', + 'CategorizerStage', + 'ModeratorStage', + 'FilterStage', + 'RankerStage', + 'PluginStage', + 'CommentFilterStage', +] \ No newline at end of file diff --git a/filter_pipeline/stages/base_stage.py b/filter_pipeline/stages/base_stage.py index a35feea..3b0c509 100644 --- a/filter_pipeline/stages/base_stage.py +++ b/filter_pipeline/stages/base_stage.py @@ -14,19 +14,29 @@ class BaseStage(ABC): Each stage processes posts sequentially and can modify FilterResults. Stages are executed in order: Categorizer → Moderator → Filter → Ranker + + ``requires_ai`` marks stages that need the AI client. The engine uses it + to decide whether a filterset can run with AI disabled (offline filtersets + that only use rule/plugin/ranker stages still run; AI stages short-circuit + to the AI-disabled path so the feed is not silently blanked). """ - def __init__(self, config: Dict[str, Any], cache: Any): + requires_ai: bool = False + + def __init__(self, config: "FilterConfig", cache: Any): """ Initialize stage. Args: - config: Configuration dictionary for this stage + config: FilterConfig instance for this pipeline run cache: FilterCache instance """ self.config = config self.cache = cache - self.enabled = config.get('enabled', True) + # Stages are enabled by default; a per-stage enabled flag can be set + # by subclasses reading their own config section. FilterConfig is not a + # dict, so do not call ``config.get(...)`` here. + self.enabled = True @abstractmethod def process( diff --git a/filter_pipeline/stages/categorizer.py b/filter_pipeline/stages/categorizer.py index 22eb668..855a5c1 100644 --- a/filter_pipeline/stages/categorizer.py +++ b/filter_pipeline/stages/categorizer.py @@ -8,6 +8,7 @@ from typing import Dict, Any from datetime import datetime from .base_stage import BaseStage +from ..registry import register_stage from ..models import FilterResult, AIAnalysisResult from ..cache import FilterCache from ..ai_client import OpenRouterClient @@ -15,6 +16,7 @@ from ..ai_client import OpenRouterClient logger = logging.getLogger(__name__) +@register_stage("categorizer") class CategorizerStage(BaseStage): """ Stage 1: Categorize content and extract tags. @@ -22,6 +24,8 @@ class CategorizerStage(BaseStage): Uses AI to detect topics/categories with content-hash based caching. """ + requires_ai = True + def __init__(self, config, cache: FilterCache): super().__init__(config, cache) diff --git a/filter_pipeline/stages/comment_filter.py b/filter_pipeline/stages/comment_filter.py new file mode 100644 index 0000000..091f04c --- /dev/null +++ b/filter_pipeline/stages/comment_filter.py @@ -0,0 +1,124 @@ +"""Comment filtering stage and tree modes.""" + +import logging +from datetime import datetime +from typing import Any, Dict, List + +from .base_stage import BaseStage +from ..models import FilterResult +from ..registry import register_stage +from ..rules import apply_rules + +logger = logging.getLogger(__name__) + + +@register_stage("comment_filter") +class CommentFilterStage(BaseStage): + """Apply filterset comment rules using configured tree modes.""" + + def get_name(self) -> str: + return "CommentFilter" + + def process(self, post: Dict[str, Any], result: FilterResult) -> FilterResult: + """Post pipeline no-op; comments are filtered through filter_comments().""" + return result + + def filter_comments(self, comments: List[Dict[str, Any]], filterset_name: str) -> List[Dict[str, Any]]: + if not comments: + return [] + + filterset = self.config.get_filterset(filterset_name) or {} + rules = filterset.get("comment_rules", {}) + mode = filterset.get("comment_filter_mode", "individual") + + if not rules: + return [dict(comment) for comment in comments] + + if mode == "tree_pruning": + return self._filter_tree_pruning(comments, rules) + if mode == "score_based": + return self._filter_individual(comments, rules, extra_check=self._passes_score_rules) + if mode == "time_bound": + return self._filter_individual(comments, rules, extra_check=self._passes_time_rules) + if mode == "content_length": + return self._filter_individual(comments, rules, extra_check=self._passes_length_rules) + + return self._filter_individual(comments, rules) + + def _filter_tree_pruning(self, comments: List[Dict[str, Any]], rules: Dict[str, Any]) -> List[Dict[str, Any]]: + comment_map = {comment["uuid"]: {**comment, "children": []} for comment in comments if comment.get("uuid")} + roots = [] + for comment in comments: + uuid = comment.get("uuid") + if not uuid or uuid not in comment_map: + continue + parent_uuid = comment.get("parent_comment_uuid") + if parent_uuid and parent_uuid in comment_map: + comment_map[parent_uuid]["children"].append(comment_map[uuid]) + else: + roots.append(comment_map[uuid]) + + def prune(nodes): + pruned = [] + for node in nodes: + if self._passes_comment_rules(node, rules): + node["children"] = prune(node.get("children", [])) + pruned.append(node) + return pruned + + return self._flatten_tree(prune(roots)) + + def _filter_individual(self, comments, rules, extra_check=None): + filtered = [] + for comment in comments: + item = dict(comment) + if self._passes_comment_rules(item, rules) and (extra_check is None or extra_check(item, rules)): + filtered.append(item) + return filtered + + def _passes_comment_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool: + return apply_rules(comment, rules) + + def _passes_score_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool: + score_rules = rules.get("score", {}) + min_score = score_rules.get("min", -1000) + return comment.get("score", 0) >= min_score + + def _passes_time_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool: + time_rules = rules.get("timestamp", {}) + timestamp = comment.get("timestamp") + if not timestamp: + return "timestamp" not in rules + try: + if isinstance(timestamp, (int, float)): + comment_time = datetime.fromtimestamp(timestamp) + else: + comment_time = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) + after = time_rules.get("after") + before = time_rules.get("before") + if after and comment_time <= datetime.fromisoformat(str(after).replace("Z", "+00:00")): + return False + if before and comment_time >= datetime.fromisoformat(str(before).replace("Z", "+00:00")): + return False + return True + except (ValueError, TypeError): + return False + + def _passes_length_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool: + length_rules = rules.get("content_length", {}) + content_length = len(comment.get("content", "")) + min_length = length_rules.get("min", 0) + max_length = length_rules.get("max", float("inf")) + return min_length <= content_length <= max_length + + def _flatten_tree(self, tree: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + flat = [] + + def traverse(nodes): + for node in nodes: + children = node.pop("children", []) + flat.append(node) + traverse(children) + + traverse(tree) + return flat \ No newline at end of file diff --git a/filter_pipeline/stages/filter.py b/filter_pipeline/stages/filter.py index 72804a0..68ee9e6 100644 --- a/filter_pipeline/stages/filter.py +++ b/filter_pipeline/stages/filter.py @@ -4,14 +4,17 @@ Apply filterset rules to posts (no AI needed - fast rule evaluation). """ import logging -from typing import Dict, Any, List +from typing import Dict, Any from .base_stage import BaseStage from ..models import FilterResult +from ..registry import register_stage +from ..rules import apply_rules logger = logging.getLogger(__name__) +@register_stage("filter") class FilterStage(BaseStage): """ Stage 3: Apply filterset rules. @@ -24,148 +27,20 @@ class FilterStage(BaseStage): return "Filter" def process(self, post: Dict[str, Any], result: FilterResult) -> FilterResult: - """ - Apply filterset rules to post. - - Args: - post: Post data - result: Current FilterResult - - Returns: - Updated FilterResult (may be rejected) - """ - # Get filterset configuration + """Apply filterset post rules to a post.""" filterset = self.config.get_filterset(result.filterset_name) - if not filterset: logger.warning(f"Filterset '{result.filterset_name}' not found") return result - # Apply post rules - post_rules = filterset.get('post_rules', {}) - if not self._evaluate_rules(post, result, post_rules): + item = dict(post) + if result.moderation_data: + item["moderation"] = result.moderation_data + + if not apply_rules(item, filterset.get("post_rules", {})): result.passed = False logger.debug(f"Filter: Post {post.get('uuid', '')} rejected by filterset rules") return result - # Post passed all rules logger.debug(f"Filter: Post {post.get('uuid', '')} passed filterset '{result.filterset_name}'") - return result - - def _evaluate_rules( - self, - post: Dict[str, Any], - result: FilterResult, - rules: Dict[str, Any] - ) -> bool: - """ - Evaluate all rules for a post. - - Returns: - True if post passes all rules, False otherwise - """ - for field, condition in rules.items(): - if not self._evaluate_condition(post, result, field, condition): - logger.debug(f"Filter: Failed condition '{field}': {condition}") - return False - - return True - - def _evaluate_condition( - self, - post: Dict[str, Any], - result: FilterResult, - field: str, - condition: Any - ) -> bool: - """ - Evaluate a single condition. - - Supported conditions: - - {"equals": value} - - {"not_equals": value} - - {"in": [values]} - - {"not_in": [values]} - - {"min": value} - - {"max": value} - - {"includes_any": [values]} - - {"excludes": [values]} - - Args: - post: Post data - result: FilterResult with moderation data - field: Field path (e.g., "score", "moderation.flags.is_safe") - condition: Condition dict - - Returns: - True if condition passes - """ - # Get field value - value = self._get_field_value(post, result, field) - - # Evaluate condition - if isinstance(condition, dict): - for op, expected in condition.items(): - if op == 'equals': - if value != expected: - return False - elif op == 'not_equals': - if value == expected: - return False - elif op == 'in': - if value not in expected: - return False - elif op == 'not_in': - if value in expected: - return False - elif op == 'min': - if value < expected: - return False - elif op == 'max': - if value > expected: - return False - elif op == 'includes_any': - # Check if any expected value is in the field (for lists) - if not isinstance(value, list): - return False - if not any(item in value for item in expected): - return False - elif op == 'excludes': - # Check that none of the excluded values are present - if isinstance(value, list): - if any(item in expected for item in value): - return False - elif value in expected: - return False - else: - logger.warning(f"Unknown condition operator: {op}") - - return True - - def _get_field_value(self, post: Dict[str, Any], result: FilterResult, field: str): - """ - Get field value from post or result. - - Supports nested fields like "moderation.flags.is_safe" - """ - parts = field.split('.') - - # Check if field is in moderation data - if parts[0] == 'moderation' and result.moderation_data: - value = result.moderation_data - for part in parts[1:]: - if isinstance(value, dict): - value = value.get(part) - else: - return None - return value - - # Check post data - value = post - for part in parts: - if isinstance(value, dict): - value = value.get(part) - else: - return None - - return value + return result \ No newline at end of file diff --git a/filter_pipeline/stages/moderator.py b/filter_pipeline/stages/moderator.py index 504bb7f..b25b801 100644 --- a/filter_pipeline/stages/moderator.py +++ b/filter_pipeline/stages/moderator.py @@ -8,6 +8,7 @@ from typing import Dict, Any from datetime import datetime from .base_stage import BaseStage +from ..registry import register_stage from ..models import FilterResult, AIAnalysisResult from ..cache import FilterCache from ..ai_client import OpenRouterClient @@ -15,6 +16,7 @@ from ..ai_client import OpenRouterClient logger = logging.getLogger(__name__) +@register_stage("moderator") class ModeratorStage(BaseStage): """ Stage 2: Content moderation and quality analysis. @@ -22,6 +24,8 @@ class ModeratorStage(BaseStage): Uses AI to analyze safety, quality, and sentiment with content-hash based caching. """ + requires_ai = True + def __init__(self, config, cache: FilterCache): super().__init__(config, cache) diff --git a/filter_pipeline/stages/plugins.py b/filter_pipeline/stages/plugins.py new file mode 100644 index 0000000..b419443 --- /dev/null +++ b/filter_pipeline/stages/plugins.py @@ -0,0 +1,89 @@ +"""Plugin consumer stage for registered BaseFilterPlugin implementations.""" + +import logging +from typing import Any, Dict, List + +from .base_stage import BaseStage +from ..models import FilterResult +from ..registry import get_plugin_class, register_stage + +logger = logging.getLogger(__name__) + + +@register_stage("plugins") +class PluginStage(BaseStage): + """Run configured filter plugins against each post.""" + + def __init__(self, config, cache): + super().__init__(config, cache) + self._plugin_instances = None + + def get_name(self) -> str: + return "Plugins" + + def process(self, post: Dict[str, Any], result: FilterResult) -> FilterResult: + """Apply configured plugins to a post/result pair.""" + plugins = self._get_plugins(result.filterset_name) + if not plugins: + return result + + context = { + "filterset_name": result.filterset_name, + "categories": result.categories, + "tags": result.tags, + "moderation": result.moderation_data, + "score_breakdown": result.score_breakdown, + } + + plugin_scores = [] + for plugin in plugins: + if not plugin.is_enabled(): + continue + + try: + if plugin.should_filter(post, context): + result.passed = False + result.tags.append(f"plugin:{plugin.get_name()}:rejected") + logger.debug("Plugin %s rejected post %s", plugin.get_name(), post.get("uuid", "")) + return result + + score = plugin.score(post, context) + plugin_scores.append(score) + result.score_breakdown[f"plugin:{plugin.get_name()}"] = score + result.tags.append(f"plugin:{plugin.get_name()}") + except Exception as e: + logger.error("Plugin %s failed: %s", plugin.get_name(), e) + result.error = f"plugin:{plugin.get_name()}: {e}" + result.passed = False + return result + + if plugin_scores: + result.score_breakdown["plugins"] = sum(plugin_scores) / len(plugin_scores) + # Blend plugin judgment with the current score without replacing + # ranking completely. Ranker can still run later and overwrite the + # final score from its own weighted factors. + result.score = (result.score + result.score_breakdown["plugins"]) / 2 + + return result + + def _get_plugins(self, filterset_name: str) -> List[Any]: + if self._plugin_instances is None: + self._plugin_instances = self._build_plugin_instances() + + filterset = self.config.get_filterset(filterset_name) or {} + plugin_names = filterset.get("plugins") + if plugin_names is None: + plugin_names = self.config.config.get("plugins", {}).get("enabled", []) + + return [self._plugin_instances[name] for name in plugin_names if name in self._plugin_instances] + + def _build_plugin_instances(self) -> Dict[str, Any]: + plugin_config = self.config.config.get("plugins", {}) + instances = {} + for name, settings in plugin_config.get("configs", {}).items(): + plugin_cls = get_plugin_class(name) + if not plugin_cls: + logger.warning("Configured plugin '%s' is not registered", name) + continue + instances[name] = plugin_cls(settings or {}) + return instances \ No newline at end of file diff --git a/filter_pipeline/stages/ranker.py b/filter_pipeline/stages/ranker.py index 69c5232..9c06304 100644 --- a/filter_pipeline/stages/ranker.py +++ b/filter_pipeline/stages/ranker.py @@ -8,11 +8,13 @@ from typing import Dict, Any from datetime import datetime from .base_stage import BaseStage +from ..registry import register_stage from ..models import FilterResult logger = logging.getLogger(__name__) +@register_stage("ranker") class RankerStage(BaseStage): """ Stage 4: Score and rank posts. diff --git a/filtersets.json b/filtersets.json index 82f8f9c..9429ea0 100644 --- a/filtersets.json +++ b/filtersets.json @@ -68,5 +68,15 @@ "moderation.flags.is_blocked": {"equals": false} }, "comment_filter_mode": "tree_pruning" + }, + "quality_filter": { + "description": "Offline quality + keyword filter (no AI required)", + "pipeline_stages": ["plugins", "ranker"], + "plugins": ["keyword", "quality"], + "post_rules": {}, + "comment_rules": { + "content": {"min_length": 5} + }, + "comment_filter_mode": "individual" } } diff --git a/generate_html.py b/generate_html.py deleted file mode 100644 index f3e85ea..0000000 --- a/generate_html.py +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env python3 -""" -Static HTML Generator -Generates static HTML from collected posts/comments with filtering and moderation. -""" - -import argparse -import json -from pathlib import Path -from datetime import datetime -from typing import Dict, List, Optional - -from filter_lib import filter_lib, TreePruningMode, IndividualFilterMode -from comment_lib import comment_lib -from html_generation_lib import html_generation_lib - - -class HTMLGenerator: - """Generate static HTML from filtered posts and comments""" - - def __init__(self, data_dir: str = "./data", filtersets_path: str = "./filtersets.json"): - self.data_dir = Path(data_dir) - self.filtersets_path = filtersets_path - - # Load filtersets - self.filtersets = filter_lib.load_filterset(filtersets_path) - - # Load moderation data into memory for faster access - self.moderation_data = self._load_all_moderation() - - def _load_all_moderation(self) -> Dict: - """Load all moderation files into a dict keyed by UUID""" - moderation_dir = self.data_dir / "moderation" - moderation_data = {} - - if moderation_dir.exists(): - for mod_file in moderation_dir.glob("*.json"): - mod_uuid = mod_file.stem - with open(mod_file, 'r') as f: - moderation_data[mod_uuid] = json.load(f) - - return moderation_data - - def _load_post_index(self) -> Dict: - """Load post index""" - index_file = self.data_dir / "post_index.json" - if index_file.exists(): - with open(index_file, 'r') as f: - return json.load(f) - return {} - - def _load_post_by_uuid(self, post_uuid: str) -> Optional[Dict]: - """Load a post by UUID""" - return filter_lib.load_data_by_uuid(post_uuid, str(self.data_dir / "posts")) - - def generate(self, filterset_name: str, theme_name: str, output_dir: str): - """ - Main generation function. - Loads data, applies filters, renders HTML. - """ - print(f"\n{'='*60}") - print(f"Generating HTML") - print(f" Filterset: {filterset_name}") - print(f" Theme: {theme_name}") - print(f" Output: {output_dir}") - print(f"{'='*60}\n") - - # Load filterset - if filterset_name not in self.filtersets: - print(f"Error: Filterset '{filterset_name}' not found") - return - - filterset = self.filtersets[filterset_name] - post_rules = filterset.get('post_rules', {}) - comment_rules = filterset.get('comment_rules', {}) - comment_filter_mode = filterset.get('comment_filter_mode', 'tree_pruning') - - # Choose comment filter mode - if comment_filter_mode == 'tree_pruning': - comment_filter = TreePruningMode - else: - comment_filter = IndividualFilterMode - - # Load theme - try: - theme = html_generation_lib.load_theme(theme_name) - except Exception as e: - print(f"Error loading theme: {e}") - return - - # Load post index - post_index = self._load_post_index() - print(f"Found {len(post_index)} posts in index") - - # Filter and render posts - filtered_posts = [] - generation_stats = { - 'total_posts_checked': 0, - 'posts_passed': 0, - 'posts_failed': 0, - 'total_comments_checked': 0, - 'comments_passed': 0, - 'comments_failed': 0 - } - - for post_id, post_uuid in post_index.items(): - generation_stats['total_posts_checked'] += 1 - - # Load post - post = self._load_post_by_uuid(post_uuid) - if not post: - continue - - # Merge moderation data - filter_lib.merge_moderation(post, self.moderation_data) - - # Apply post rules - if not filter_lib.apply_rules(post, post_rules): - generation_stats['posts_failed'] += 1 - continue - - generation_stats['posts_passed'] += 1 - - # Load comments for this post - comments = comment_lib.load_comments_for_post(post_uuid, str(self.data_dir)) - - if comments: - generation_stats['total_comments_checked'] += len(comments) - - # Filter comments using selected mode - filtered_comments = comment_filter.filter(comments, comment_rules, self.moderation_data) - generation_stats['comments_passed'] += len(filtered_comments) - generation_stats['comments_failed'] += len(comments) - len(filtered_comments) - - # Build comment tree for rendering - comment_tree = comment_lib.build_comment_tree(filtered_comments) - post['comments'] = comment_tree - else: - post['comments'] = [] - - filtered_posts.append(post) - - print(f"\nFiltering Results:") - print(f" Posts: {generation_stats['posts_passed']}/{generation_stats['total_posts_checked']} passed") - print(f" Comments: {generation_stats['comments_passed']}/{generation_stats['total_comments_checked']} passed") - - # Create output directory - output_path = Path(output_dir) / filterset_name - output_path.mkdir(parents=True, exist_ok=True) - - # Render index page - for post in filtered_posts: - post['post_url'] = f"{post['uuid']}.html" - index_html = html_generation_lib.render_index(filtered_posts, theme, filterset_name) - html_generation_lib.write_html_file(index_html, str(output_path / "index.html")) - - # Render individual post pages - for post in filtered_posts: - post_html = html_generation_lib.render_post_page(post, theme, post.get('comments')) - post_filename = f"{post['uuid']}.html" - html_generation_lib.write_html_file(post_html, str(output_path / post_filename)) - - # Generate metadata file - metadata = { - "generated_at": datetime.now().isoformat(), - "filterset": filterset_name, - "filterset_config": filterset, - "theme": theme_name, - "output_directory": str(output_path), - "statistics": { - **generation_stats, - "posts_generated": len(filtered_posts) - }, - "comment_filter_mode": comment_filter_mode - } - - metadata_file = output_path / "metadata.json" - with open(metadata_file, 'w') as f: - json.dump(metadata, f, indent=2) - - print(f"\nGeneration Complete:") - print(f" Index page: {output_path / 'index.html'}") - print(f" Individual posts: {len(filtered_posts)} files") - print(f" Metadata: {metadata_file}") - print(f"{'='*60}\n") - - -def interactive_mode(): - """Interactive mode for human use""" - print("\n=== HTML Generator - Interactive Mode ===\n") - - # List available filtersets - try: - filtersets = filter_lib.load_filterset("./filtersets.json") - print("Available filtersets:") - for i, (name, config) in enumerate(filtersets.items(), 1): - desc = config.get('description', 'No description') - print(f" {i}. {name} - {desc}") - - filterset_choice = input("\nEnter filterset name or number: ").strip() - - # Handle numeric choice - if filterset_choice.isdigit(): - idx = int(filterset_choice) - 1 - filterset_name = list(filtersets.keys())[idx] - else: - filterset_name = filterset_choice - - # List available themes - themes_dir = Path("./themes") - if themes_dir.exists(): - themes = [d.name for d in themes_dir.iterdir() if d.is_dir()] - print("\nAvailable themes:") - for i, theme in enumerate(themes, 1): - print(f" {i}. {theme}") - - theme_choice = input("\nEnter theme name or number: ").strip() - - if theme_choice.isdigit(): - idx = int(theme_choice) - 1 - theme_name = themes[idx] - else: - theme_name = theme_choice - else: - theme_name = "vanilla-js" - - # Output directory - output_dir = input("\nOutput directory [./active_html]: ").strip() - if not output_dir: - output_dir = "./active_html" - - # Run generation - generator = HTMLGenerator() - generator.generate(filterset_name, theme_name, output_dir) - - except Exception as e: - print(f"Error: {e}") - import traceback - traceback.print_exc() - - -def main(): - """Main entry point with CLI argument parsing""" - parser = argparse.ArgumentParser( - description="Generate static HTML from collected posts with filtering" - ) - - parser.add_argument( - '--filterset', - default='safe_content', - help='Filterset name to use (default: safe_content)' - ) - - parser.add_argument( - '--theme', - default='vanilla-js', - help='Theme name to use (default: vanilla-js)' - ) - - parser.add_argument( - '--output', - default='./active_html', - help='Output directory (default: ./active_html)' - ) - - parser.add_argument( - '--interactive', - action='store_true', - help='Run in interactive mode' - ) - - parser.add_argument( - '--data-dir', - default='./data', - help='Data directory (default: ./data)' - ) - - parser.add_argument( - '--filtersets-file', - default='./filtersets.json', - help='Filtersets file (default: ./filtersets.json)' - ) - - args = parser.parse_args() - - if args.interactive: - interactive_mode() - else: - generator = HTMLGenerator( - data_dir=args.data_dir, - filtersets_path=args.filtersets_file - ) - generator.generate(args.filterset, args.theme, args.output) - - -if __name__ == "__main__": - main() diff --git a/html_generation_lib.py b/html_generation_lib.py deleted file mode 100644 index c5dc478..0000000 --- a/html_generation_lib.py +++ /dev/null @@ -1,515 +0,0 @@ -""" -HTML Generation Library -Atomic functions for loading themes and rendering HTML from templates. -""" - -import json -from pathlib import Path -from typing import Dict, List, Optional -from datetime import datetime -import jinja2 - - -class html_generation_lib: - """Atomic HTML generation functions""" - - @staticmethod - def load_theme(theme_name: str, themes_dir: str = './themes') -> Dict: - """ - Load theme configuration and templates. - - Returns: - Dict with theme config, template paths, and metadata - """ - theme_dir = Path(themes_dir) / theme_name - theme_config_path = theme_dir / 'theme.json' - - if not theme_config_path.exists(): - raise FileNotFoundError(f"Theme config not found: {theme_config_path}") - - with open(theme_config_path, 'r') as f: - config = json.load(f) - - # Load template files - templates = {} - if 'templates' in config: - for template_name, template_path in config['templates'].items(): - full_path = Path(template_path) - if full_path.exists(): - with open(full_path, 'r') as f: - templates[template_name] = f.read() - - config['loaded_templates'] = templates - config['theme_dir'] = str(theme_dir) - - return config - - @staticmethod - def render_template(template_string: str, data: Dict) -> str: - """ - Render template string with data using Jinja2 templating. - Handles nested expressions and complex logic better. - - Args: - template_string: Template with {{variable}} placeholders - data: Dict of data to inject - - Returns: - Rendered HTML string - """ - # Add helper functions to data context - context = { - **data, - 'formatTime': html_generation_lib.format_time, - 'formatTimeAgo': html_generation_lib.format_time_ago, - 'formatDateTime': html_generation_lib.format_datetime, - 'truncate': html_generation_lib.truncate, - 'renderMarkdown': html_generation_lib.render_markdown, - 'escapeHtml': html_generation_lib.escape_html - } - - # Extract template content from ', template_string, re.DOTALL) - if match: - template_string = match.group(1) - - # Use Jinja2 for template rendering - try: - template = jinja2.Template(template_string) - return template.render(**context) - except Exception as e: - print(f"Template rendering error: {e}") - return f"" - - @staticmethod - def render_post(post: Dict, theme: Dict, comments: Optional[List[Dict]] = None) -> str: - """ - Render single post to HTML using theme's post/card/detail template. - - Args: - post: Post data dict - theme: Theme config with loaded templates - comments: Optional list of comments to render with post - - Returns: - Rendered HTML string - """ - # Choose template (prefer 'detail' if comments, else 'card') - template_name = 'detail' if comments else 'card' - if template_name not in theme.get('loaded_templates', {}): - template_name = 'card' # Fallback - - template = theme['loaded_templates'].get(template_name) - if not template: - return f"" - - # Render comments if provided - comments_section = '' - if comments: - comments_section = html_generation_lib.render_comment_tree(comments, theme) - - # Create post data with comments_section - post_data = dict(post) - post_data['comments_section'] = comments_section - - # Render post - return html_generation_lib.render_template(template, post_data) - - @staticmethod - def render_post_page(post: Dict, theme: Dict, comments: Optional[List[Dict]] = None) -> str: - """ - Render single post as a complete HTML page with navigation. - - Args: - post: Post data dict - theme: Theme config with loaded templates - comments: Optional list of comments to render with post - - Returns: - Complete HTML page string - """ - # Render the post content - post_content = html_generation_lib.render_post(post, theme, comments) - - # Build CSS links - css_links = '' - if theme.get('css_dependencies'): - for css_path in theme['css_dependencies']: - adjusted_path = css_path.replace('./themes/', '../../themes/') - css_links += f' \n' - - # Build JS scripts - js_scripts = '' - if theme.get('js_dependencies'): - for js_path in theme['js_dependencies']: - adjusted_path = js_path.replace('./themes/', '../../themes/') - js_scripts += f' \n' - - # Create full page - page_html = f''' - - - - - {post.get('title', 'Post')} - BalanceBoard -{css_links} - - - - - -
- - - - -
-
- {post_content} -
-
-
-{js_scripts} - -''' - - return page_html - - @staticmethod - def render_comment_tree(comments: List[Dict], theme: Dict, depth: int = 0) -> str: - """ - Recursively render nested comment tree (unlimited depth). - - Args: - comments: List of comment dicts (may have 'children') - theme: Theme config with loaded templates - depth: Current nesting depth - - Returns: - Rendered HTML string for all comments - """ - if not comments: - return '' - - template = theme['loaded_templates'].get('comment') - if not template: - return '' - - html_parts = [] - - for comment in comments: - # Recursively render children first - children = comment.get('children', []) - if children: - children_html = html_generation_lib.render_comment_tree(children, theme, depth + 1) - else: - children_html = '' - - # Add depth and children_section to comment data - comment_data = {**comment, 'depth': depth, 'children_section': children_html} - - # Render this comment - comment_html = html_generation_lib.render_template(template, comment_data) - - html_parts.append(comment_html) - - return '\n'.join(html_parts) - - @staticmethod - def render_index(posts: List[Dict], theme: Dict, filterset_name: str = '') -> str: - """ - Render index/list page with all posts. - - Args: - posts: List of post dicts - theme: Theme config with loaded templates - filterset_name: Name of filterset used (for display) - - Returns: - Complete HTML page - """ - template = theme['loaded_templates'].get('list') or theme['loaded_templates'].get('card') - if not template: - return '' - - # Render each post - post_items = [] - for post in posts: - # Update post URL to use Flask route - post_data = dict(post) - post_data['post_url'] = f"/post/{post['uuid']}" - post_html = html_generation_lib.render_template(template, post_data) - post_items.append(post_html) - - # Create full page - css_links = '' - if theme.get('css_dependencies'): - for css_path in theme['css_dependencies']: - # Adjust relative paths to work from subdirectories (e.g., active_html/no_filter/) - # Convert ./themes/... to ../../themes/... - adjusted_path = css_path.replace('./themes/', '../../themes/') - css_links += f' \n' - - js_scripts = '' - if theme.get('js_dependencies'): - for js_path in theme['js_dependencies']: - # Adjust relative paths to work from subdirectories - adjusted_path = js_path.replace('./themes/', '../../themes/') - js_scripts += f' \n' - - page_html = f''' - - - - - BalanceBoard - Content Feed -{css_links} - - - - - -
- - - - -
-
-
-

{filterset_name.replace('_', ' ').title() if filterset_name else 'All Posts'}

-

{len(posts)} posts

-
-
- {''.join(post_items)} -
-
-
-
- -''' - - return page_html - - @staticmethod - def write_html_file(html: str, output_path: str) -> None: - """ - Write HTML string to file. - - Args: - html: HTML content - output_path: File path to write to - """ - output_file = Path(output_path) - output_file.parent.mkdir(parents=True, exist_ok=True) - - with open(output_file, 'w', encoding='utf-8') as f: - f.write(html) - - # Helper functions for templates - - @staticmethod - def format_time(timestamp: int) -> str: - """Format timestamp as time""" - dt = datetime.fromtimestamp(timestamp) - return dt.strftime('%H:%M') - - @staticmethod - def format_time_ago(timestamp: int) -> str: - """Format timestamp as relative time (e.g., '2 hours ago')""" - now = datetime.now() - dt = datetime.fromtimestamp(timestamp) - diff = now - dt - - seconds = diff.total_seconds() - if seconds < 60: - return 'just now' - elif seconds < 3600: - minutes = int(seconds / 60) - return f'{minutes} minute{"s" if minutes != 1 else ""} ago' - elif seconds < 86400: - hours = int(seconds / 3600) - return f'{hours} hour{"s" if hours != 1 else ""} ago' - elif seconds < 604800: - days = int(seconds / 86400) - return f'{days} day{"s" if days != 1 else ""} ago' - else: - weeks = int(seconds / 604800) - return f'{weeks} week{"s" if weeks != 1 else ""} ago' - - @staticmethod - def format_datetime(timestamp: int) -> str: - """Format timestamp as full datetime""" - dt = datetime.fromtimestamp(timestamp) - return dt.strftime('%B %d, %Y at %H:%M') - - @staticmethod - def truncate(text: str, max_length: int) -> str: - """Truncate text to max length""" - if len(text) <= max_length: - return text - return text[:max_length].strip() + '...' - - @staticmethod - def render_markdown(text: str) -> str: - """Basic markdown rendering""" - if not text: - return '' - - # Basic markdown conversions - html = text - html = html.replace('&', '&').replace('<', '<').replace('>', '>') - html = html.replace('\n\n', '

') - html = html.replace('\n', '
') - - # Bold and italic - import re - html = re.sub(r'\*\*(.*?)\*\*', r'\1', html) - html = re.sub(r'\*(.*?)\*', r'\1', html) - - # Images (must be processed before links since they use similar syntax) - html = re.sub(r'!\[(.*?)\]\((.*?)\)', r'\1', html) - - # Links - html = re.sub(r'\[(.*?)\]\((.*?)\)', r'\1', html) - - return f'

{html}

' - - @staticmethod - def escape_html(text: str) -> str: - """Escape HTML entities""" - return (text - .replace('&', '&') - .replace('<', '<') - .replace('>', '>') - .replace('"', '"') - .replace("'", ''')) diff --git a/migrate_content_to_db.py b/migrate_content_to_db.py new file mode 100644 index 0000000..d787cdb --- /dev/null +++ b/migrate_content_to_db.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python +"""Backfill on-disk content JSON into the Postgres ``posts``/``comments`` tables. + +Phase 3 prep (Agent D): reads ``data/posts/*.json`` and ``data/comments/*.json`` +and upserts them into the ``Post`` / ``Comment`` models. Live reads/writes still +go through ``PostService`` (disk JSON) — this script only populates the DB so a +later cutover has data to read. It is idempotent: existing rows are skipped by +``uuid`` (run it again after collecting new content to backfill only the new +files). + +Usage:: + + python migrate_content_to_db.py [--data-dir data] [--batch-size 500] [--dry-run] + +Requires the full Flask/Postgres stack (``DATABASE_URL`` or the POSTGRES_* +env vars) — run inside the docker compose environment, not locally. +""" + +import argparse +import json +import logging +import sys +from pathlib import Path + +from app import create_app +from database import db +from models import Comment, Post + +logging.basicConfig( + level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" +) +logger = logging.getLogger("migrate_content_to_db") + + +def _load_json(path: Path): + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError) as e: + logger.warning("Skipping unreadable file %s: %s", path, e) + return None + + +def _existing_uuids(model, uuids): + """Return the subset of ``uuids`` already present in the table.""" + if not uuids: + return set() + found = set() + # Chunk to avoid huge IN clauses. + for i in range(0, len(uuids), 500): + chunk = uuids[i:i + 500] + rows = db.session.query(model.uuid).filter(model.uuid.in_(chunk)).all() + found.update(r[0] for r in rows) + return found + + +def backfill_posts(posts_dir: Path, batch_size: int, dry_run: bool) -> int: + files = sorted(posts_dir.glob("*.json")) if posts_dir.exists() else [] + if not files: + logger.info("No post files found in %s", posts_dir) + return 0 + + records = [] + for pf in files: + data = _load_json(pf) + if not data or not data.get("uuid"): + continue + records.append(data) + + seen = _existing_uuids(Post, [r["uuid"] for r in records]) + inserted = 0 + batch = [] + for r in records: + if r["uuid"] in seen: + continue + batch.append(Post( + uuid=r["uuid"], + external_id=r.get("id"), + platform=r.get("platform", "") or "", + source=r.get("source", "") or "", + title=(r.get("title") or "")[:500], + author=r.get("author"), + url=r.get("url"), + content=r.get("content"), + score=int(r.get("score", 0) or 0), + timestamp=int(r.get("timestamp", 0) or 0), + tags=r.get("tags"), + moderation_uuid=r.get("moderation_uuid"), + )) + inserted += 1 + if len(batch) >= batch_size: + _flush(batch, dry_run) + batch = [] + _flush(batch, dry_run) + logger.info("Posts: backfilled %d new (%d already present)", inserted, len(seen)) + return inserted + + +def backfill_comments(comments_dir: Path, batch_size: int, dry_run: bool) -> int: + files = sorted(comments_dir.glob("*.json")) if comments_dir.exists() else [] + if not files: + logger.info("No comment files found in %s", comments_dir) + return 0 + + records = [] + for cf in files: + data = _load_json(cf) + if not data or not data.get("uuid"): + continue + records.append(data) + + seen = _existing_uuids(Comment, [r["uuid"] for r in records]) + inserted = 0 + batch = [] + for r in records: + if r["uuid"] in seen: + continue + batch.append(Comment( + uuid=r["uuid"], + post_uuid=r.get("post_uuid") or "", + platform=r.get("platform"), + parent_comment_uuid=r.get("parent_comment_uuid"), + comment_id=r.get("comment_id"), + author=r.get("author"), + content=r.get("content"), + score=int(r.get("score", 0) or 0), + timestamp=int(r.get("timestamp", 0) or 0), + depth=int(r.get("depth", 0) or 0), + moderation_uuid=r.get("moderation_uuid"), + )) + inserted += 1 + if len(batch) >= batch_size: + _flush(batch, dry_run) + batch = [] + _flush(batch, dry_run) + logger.info("Comments: backfilled %d new (%d already present)", inserted, len(seen)) + return inserted + + +def _flush(batch, dry_run): + if not batch: + return + if dry_run: + logger.info("[dry-run] would insert %d rows", len(batch)) + return + db.session.bulk_save_objects(batch) + db.session.commit() + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Backfill content JSON into Postgres.") + parser.add_argument("--data-dir", default="data", help="Root data directory.") + parser.add_argument("--batch-size", type=int, default=500, help="Insert batch size.") + parser.add_argument("--dry-run", action="store_true", help="Log counts without writing.") + args = parser.parse_args(argv) + + app = create_app() + with app.app_context(): + posts_dir = Path(args.data_dir) / "posts" + comments_dir = Path(args.data_dir) / "comments" + p = backfill_posts(posts_dir, args.batch_size, args.dry_run) + c = backfill_comments(comments_dir, args.batch_size, args.dry_run) + logger.info("Done. posts=%d comments=%d", p, c) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/models.py b/models.py index 4cd07d9..8058560 100644 --- a/models.py +++ b/models.py @@ -246,3 +246,61 @@ class Bookmark(db.Model): def __repr__(self): return f'' + + +class Post(db.Model): + """A collected post/item, mirroring the on-disk ``data/posts/*.json`` schema. + + Phase 3 prep: the model and ``migrate_content_to_db.py`` backfill exist, but + live reads/writes still go through ``PostService`` (disk JSON). The cutover + is gated on Phase 2 filter behavior being stable — see ``progress.md`` and + ``parallel.md`` (Agent D). Do not point the API at this model yet. + """ + + __tablename__ = 'posts' + + uuid = db.Column(db.String(64), primary_key=True) + external_id = db.Column(db.String(255), nullable=True, index=True) + platform = db.Column(db.String(50), nullable=False, index=True) + source = db.Column(db.String(500), nullable=False, default='', index=True) + title = db.Column(db.String(500), nullable=False, default='') + author = db.Column(db.String(255), nullable=True) + url = db.Column(db.Text, nullable=True) + content = db.Column(db.Text, nullable=True) + score = db.Column(db.Integer, nullable=False, default=0) + # Unix epoch seconds; BigInteger so far-future/large values fit. + timestamp = db.Column(db.BigInteger, nullable=False, default=0, index=True) + tags = db.Column(db.JSON, nullable=True) + moderation_uuid = db.Column(db.String(64), nullable=True, index=True) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + + comments = db.relationship('Comment', backref='post', lazy='dynamic') + + def __repr__(self): + return f'' + + +class Comment(db.Model): + """A comment on a post, mirroring ``data/comments/*.json``. + + ``parent_comment_uuid`` is a self-reference implementing the comment tree; + null marks a top-level comment. ``comment_id`` is the platform's own id. + """ + + __tablename__ = 'comments' + + uuid = db.Column(db.String(64), primary_key=True) + post_uuid = db.Column(db.String(64), db.ForeignKey('posts.uuid'), nullable=False, index=True) + platform = db.Column(db.String(50), nullable=True, index=True) + parent_comment_uuid = db.Column(db.String(64), nullable=True, index=True) + comment_id = db.Column(db.String(100), nullable=True) + author = db.Column(db.String(255), nullable=True) + content = db.Column(db.Text, nullable=True) + score = db.Column(db.Integer, nullable=False, default=0) + timestamp = db.Column(db.BigInteger, nullable=False, default=0, index=True) + depth = db.Column(db.Integer, nullable=False, default=0) + moderation_uuid = db.Column(db.String(64), nullable=True, index=True) + created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + + def __repr__(self): + return f'' diff --git a/parallel.md b/parallel.md new file mode 100644 index 0000000..9e067f0 --- /dev/null +++ b/parallel.md @@ -0,0 +1,57 @@ +# Parallel Work Handoff + +Current status: Phase 1 is mostly complete. `app.py` is now a `create_app()` +factory, `/api/v1` lives in `blueprints/api.py`, and legacy Jinja route groups +are split under `routes/`. Runtime verification is still pending because the +local environment does not have the full Flask/Postgres stack available. + +## Good Parallel Workstreams + +### Agent A: Phase 1 Verification + Cleanup +- Owns: `app.py`, `run_app.py`, `start_server.py`, `Dockerfile`, README/DEPLOYMENT. +- Verify `flask --app app:create_app routes` and startup in a deps-installed environment. +- Check Auth0 registration under repeated `create_app()` calls. +- Confirm `url_for(...)` endpoints still exist for templates. +- Remove stale compatibility notes from `progress.md` once verified. +- Avoid changing filter pipeline internals. + +### Agent B: Phase 2 Filter Registry +- Owns: `filter_pipeline/`, `filter_config.json`, `filtersets.json`. +- Add stage/plugin registry and auto-discovery. +- Replace hardcoded stage dicts in `filter_pipeline/engine.py`. +- Bridge `BaseFilterPlugin` to live `FilterResult` handling. +- Avoid editing route modules except for minimal API integration points. + +### Agent C: Comment Filtering Consolidation +- Owns: `comment_lib.py`, `filter_lib.py`, comment-related pipeline stages. +- Port comment tree modes into `filter_pipeline/stages/comment_filter.py`. +- Wire filtered comments into `/api/v1/posts/` and `/api/v1/comments/`. +- Add small sample-data tests if a test harness is added. +- Coordinate with Agent B on registry names and stage contracts. + +### Agent D: Phase 3 Data Model Prep +- Owns: `models.py`, new migration scripts, DB query services. +- Design `Post` and `Comment` SQLAlchemy models matching current JSON schema. +- Draft `migrate_content_to_db.py` backfill from `data/posts` and `data/comments`. +- Do not switch live reads/writes until Phase 2 route/filter behavior is stable. + +### Agent E: Test Harness +- Owns: `tests/`, pytest config, lightweight fixtures. +- Add app-factory tests that assert route registration and endpoint names. +- Add `/api/v1` contract tests with monkeypatched `post_service` and `get_filter_engine`. +- Keep tests independent of a live Postgres where possible. + +## Serialization Points + +- Do Phase 1 runtime verification before deleting any more legacy UI behavior. +- Phase 2 registry work and comment filtering can happen together, but merge the registry contract first. +- Phase 3 database cutover should wait until `/api/v1/posts` and comment filtering behavior is stable. +- Phase 4 SPA work can scaffold independently, but feature parity work depends on stable `/api/v1` contracts. + +## Shared Cautions + +- Preserve endpoint names used by templates until the SPA replaces them. +- Do not delete Jinja templates in this pass. +- Do not remove JSON file reads until Postgres backfill and DB query paths are verified. +- Run at least `python -m py_compile` on touched Python files. +- Update `progress.md` after each substantial change. diff --git a/platforms/__init__.py b/platforms/__init__.py new file mode 100644 index 0000000..f13efcd --- /dev/null +++ b/platforms/__init__.py @@ -0,0 +1,19 @@ +"""Platform fetcher extension points.""" + +from .base import PlatformFetcher +from .registry import ( + discover_modules, + get_platform_class, + get_platform_fetcher, + get_registered_platforms, + register_platform, +) + +__all__ = [ + "PlatformFetcher", + "discover_modules", + "get_platform_class", + "get_platform_fetcher", + "get_registered_platforms", + "register_platform", +] diff --git a/platforms/base.py b/platforms/base.py new file mode 100644 index 0000000..e3758f3 --- /dev/null +++ b/platforms/base.py @@ -0,0 +1,18 @@ +"""Base protocol for platform fetchers.""" + +from typing import Dict, List, Protocol + + +class PlatformFetcher(Protocol): + """Fetch posts for one configured platform/community.""" + + name: str + + def fetch_posts( + self, + start_date: str, + end_date: str, + community: str, + max_posts: int, + ) -> List[Dict]: + """Return posts normalized to the existing collection schema.""" diff --git a/platforms/builtins.py b/platforms/builtins.py new file mode 100644 index 0000000..2e4e268 --- /dev/null +++ b/platforms/builtins.py @@ -0,0 +1,50 @@ +"""Built-in platform fetchers backed by the legacy fetch functions.""" + +from .registry import register_platform + + +class _LegacyMethodFetcher: + method_name = "" + + def fetch_posts(self, start_date, end_date, community, max_posts): + # Import lazily so data_collection_lib can import this module while defining data_methods. + from data_collection_lib import data_methods + + method = getattr(data_methods.fetchers, self.method_name) + return method(start_date, end_date, community, max_posts) + + +@register_platform("reddit") +class RedditFetcher(_LegacyMethodFetcher): + name = "reddit" + method_name = "getRedditData" + + +@register_platform("pushshift") +class PushshiftFetcher(_LegacyMethodFetcher): + name = "pushshift" + method_name = "getPushshiftData" + + +@register_platform("hackernews") +class HackerNewsFetcher(_LegacyMethodFetcher): + name = "hackernews" + method_name = "getHackerNewsData" + + +@register_platform("lobsters") +class LobstersFetcher(_LegacyMethodFetcher): + name = "lobsters" + method_name = "getLobstersData" + + +@register_platform("stackexchange") +class StackExchangeFetcher(_LegacyMethodFetcher): + name = "stackexchange" + method_name = "getStackExchangeData" + + +@register_platform("rss") +class RSSFetcher(_LegacyMethodFetcher): + name = "rss" + method_name = "getRSSData" diff --git a/platforms/registry.py b/platforms/registry.py new file mode 100644 index 0000000..7c78011 --- /dev/null +++ b/platforms/registry.py @@ -0,0 +1,40 @@ +"""Platform fetcher registry for data collection.""" + +from importlib import import_module +from typing import Dict, Iterable, Optional, Type + +from .base import PlatformFetcher + +_PLATFORM_FETCHERS: Dict[str, Type[PlatformFetcher]] = {} + + +def register_platform(name: str): + """Register a platform fetcher class by config/platform name.""" + normalized = name.strip().lower() + if not normalized: + raise ValueError("Platform name must not be empty") + + def decorator(cls: Type[PlatformFetcher]) -> Type[PlatformFetcher]: + _PLATFORM_FETCHERS[normalized] = cls + return cls + + return decorator + + +def get_platform_class(name: str) -> Optional[Type[PlatformFetcher]]: + return _PLATFORM_FETCHERS.get((name or "").strip().lower()) + + +def get_platform_fetcher(name: str) -> Optional[PlatformFetcher]: + cls = get_platform_class(name) + return cls() if cls else None + + +def get_registered_platforms() -> Dict[str, Type[PlatformFetcher]]: + return dict(_PLATFORM_FETCHERS) + + +def discover_modules(module_names: Iterable[str]) -> None: + for module_name in module_names: + if module_name: + import_module(module_name) diff --git a/progress.md b/progress.md new file mode 100644 index 0000000..92cc233 --- /dev/null +++ b/progress.md @@ -0,0 +1,157 @@ +# BalanceBoard Refactor — Progress + +Tracking work toward `REFACTOR_GOAL.md` (with the pluginability thread from +`.claude/plans/nested-stirring-dusk.md`). Updated continuously as work proceeds. + +## Status legend +- [x] done [ ] todo [~] in progress [-] deferred / blocked + +## Phase 0 — Stop the bleeding ✅ COMMITTED +Branch: `refactor/phase-0-bugfixes` · Commit: `cdba720` +- [x] Remove import-time side effects from `app.py` (polling/filter_engine + deferred to a one-shot `before_request`) +- [x] Fix `migrate_bookmarks.py` None bug +- [x] Password min-length consistent (8) in reset route +- [x] `post_detail.html`: `timeago` filter + `data-timestamp`; escape-then-Markup + `nl2br`; drop `| safe` from comment/post content (XSS) +- [x] AI-disabled filtersets now `status=FAILED` + explicit error (not silent) +- [x] Verified no source-file mojibake (ingest encoding deferred to Phase 3) + +## Phase 1 — App factory + blueprints + `/api/v1` [~] +- [x] `config.py` (Config class + constants) +- [x] `extensions.py` (login_manager, oauth, lazy get_filter_engine/get_polling_service) +- [x] `security.py` (is_safe_filterset/is_safe_path/is_allowed_file) +- [x] `services/__init__.py` +- [x] `services/post_service.py` (cache, stats, comment tree, platform config; `app.py` wrappers now delegate here) +- [x] `services/settings_service.py` (settings parse/validate/defaults; `app.py` wrapper delegates here) +- [x] `blueprints/api.py` (Flask Blueprint mounted at `/api/v1`; includes posts, post detail, comments, platforms, bookmarks, filters) +- [x] `routes/` modules (auth, pages, settings, admin, assets extracted with endpoint names preserved) +- [x] `app.py` to `create_app()` factory (no module-level `app`; `__main__` creates a local app) +- [x] Update entrypoints (`run_app.py`, `start_server.py`, Dockerfile) to use `create_app()` / `app:create_app` +- [x] Update template/theme fetch strings `/api/*` to `/api/v1/*` (8 fetches found) +- [x] py_compile verification across the whole project (app, entrypoints, blueprints, + routes, services, config/extensions/security, models, and filter_pipeline) — passes +- [x] Static endpoint check: all 27 `url_for(...)` endpoint names used by templates + resolve to a defined route function (no missing endpoints after the route split) +- [-] Runtime verification (`flask --app app:create_app routes`, Auth0 under repeated + `create_app()`, startup) — needs `docker compose up`; static review shows the + `authlib` `oauth.register` path is idempotent (dict-based), so repeated factory + calls should not collide, but this is unconfirmed without a deps-installed env + +Latest continuation 4: extracted settings/profile/avatar routes to `routes/settings.py` +and admin/polling routes to `routes/admin.py`; removed duplicate legacy `/api/*` +routes from `app.py`; converted `app.py` to `create_app()` with no module-level +Flask app; updated `run_app.py`, `start_server.py`, Dockerfile, and README init +command to use the factory. +Latest continuation 3: extracted legacy Jinja page routes to `routes/pages.py`, +asset/static routes to `routes/assets.py`, and auth/signup/password-reset/Auth0 +routes to `routes/auth.py`; endpoint function names are preserved for existing +`url_for` calls. `app.py` now mainly retains legacy `/api/*`, settings, and admin routes. +Latest continuation 2: added `blueprints/api.py` and mounted it at `/api/v1`; +updated dashboard/bookmarks/theme fetch calls to `/api/v1/*` while leaving legacy +`/api/*` app routes in place for compatibility during the split. +Latest continuation: removed duplicate post/comment cache, platform-config helpers, +security helpers, and comment-tree builder from `app.py`; compatibility wrappers now +call `PostService`, `SettingsService`, and `security.py`. `services/__init__.py` +import is now unblocked by the new `services/settings_service.py`. +Design decision: API routes use a real Blueprint at `/api/v1` (templates call +them via fetch strings, not `url_for`). Jinja page/auth/settings/admin routes +use modular `register_*_routes(app)` that preserve original endpoint names, so +~60 template `url_for` calls need no changes (can't runtime-verify, minimizing +risk). Promoting page routes to full Blueprints is deferred until Phase 6 adds +a test harness to catch endpoint regressions. + +## Phase 2 — One pluggable filter system [x] +- [x] `filter_pipeline/registry.py` (`@register_stage`/`@register_plugin` + module discovery) +- [x] Replace hardcoded `engine.py` stage dict with registry +- [x] Port `filter_lib` operators + `comment_lib` tree modes into pipeline (`filter_pipeline/rules.py`, `stages/comment_filter.py`) +- [x] New `stages/comment_filter.py` wired into live `/api/v1/posts/` and `/api/v1/comments/` +- [x] Bridge `BaseFilterPlugin` → `FilterResult` (`stages/plugins.py` consumer stage) +- [x] `plugins`/`stages` config wiring in filter_config.json (`stage_modules`/`plugins.modules` discovery hooks plus default `plugins` stage) +- [x] Re-enable Keyword/Quality plugins via registry and plugin consumer stage +- [x] Delete filter_lib/comment_lib/html_generation_lib/generate_html/active_html route and theme template prompt path + + +Latest Phase 2 continuation 3: removed the dead generated-static path. The +`/feed/` active_html route is gone, the admin regenerate-content +route/form is gone, Docker no longer creates or mounts `/app/active_html`, and +`filter_lib.py`, `comment_lib.py`, `html_generation_lib.py`, `generate_html.py`, +and `themes/template_prompt.txt` were deleted. Focused `rg` found only planning +references afterward, and `python -m py_compile` passed for the touched app, +route, service, and filter pipeline modules. +Latest Phase 2 continuation 2: added shared rule evaluation, moved comment +filter tree modes into `filter_pipeline/stages/comment_filter.py`, wired live +`/api/v1` post-detail/comment endpoints through `FilterEngine.filter_comments()`, +and added `filter_pipeline/stages/plugins.py` so registered Keyword/Quality +plugins have a pipeline consumer. `python -m py_compile` passed for the touched +app, route, service, and filter pipeline modules after these changes. +Latest Phase 2 continuation: added a stage/plugin registry, decorated built-in +categorizer/moderator/filter/ranker stages and keyword/quality plugins, changed +`FilterEngine._init_stages()` to instantiate registered stages, and added +`pipeline.stage_modules` / `plugins.modules` discovery hooks in `filter_config.json`. +## Phase 3 — Pluggable platform fetchers + Postgres [x] +- [x] `PlatformFetcher` protocol + `@register_platform` registry +- [x] Convert if/elif dispatch → `platforms/` fetcher classes (thin adapters over existing fetch functions) +- [x] `Post`/`Comment` SQLAlchemy models + indexes (added to `models.py`; mirror the + on-disk `data/{posts,comments}/*.json` schema — uuid, platform, source, title, + external_id, author, url, content, score, timestamp, tags(JSON), moderation_uuid for posts; + uuid, post_uuid (FK→posts), platform, parent_comment_uuid (self-ref), comment_id, author, + content, score, timestamp, depth, moderation_uuid for comments) +- [x] `migrate_content_to_db.py` backfill from data/*.json (idempotent by uuid, + batched bulk insert, `--dry-run` supported; requires the docker/Postgres env) +- [x] Replace `_load_posts_cache` + directory scans with DB queries + TTL cache (`PostService` is DB-first, disk fallback only) +- [x] Fetchers write to DB; data/ becomes archive-only (`data_collection.py` upserts Post/Comment after archive JSON writes) + + +Latest Phase 3 continuation 2: cut live content access over to the DB path. +`PostService` now refreshes its TTL cache from `Post`/`Comment` queries first, +with the legacy JSON reader only as a local/dev fallback when DB content is +empty or unavailable. `latest_content_mtime()` and `source_counts()` also query +Postgres first. `data_collection.py` now upserts collected posts/comments into +Postgres after writing archive JSON, and the model/backfill mapping now preserves +post `external_id`, comment `platform`, and longer source strings. Focused +py_compile passed for app, routes, services, models, migration, collection, +platforms, and filter pipeline modules. +Latest Phase 3 continuation: confirmed existing parallel-agent DB prep +(`Post`/`Comment` models and `migrate_content_to_db.py`) and added the +platform fetcher extension point. New `platforms/` modules define the +`PlatformFetcher` protocol, `@register_platform` registry, and built-in +fetcher classes for reddit, pushshift, hackernews, lobsters, stackexchange, +and rss. `data_methods.getData()` now resolves platforms through the registry +instead of an if/elif chain while keeping the legacy network fetch functions as +implementation details. Verified with py_compile and a registry smoke test. +## Phase 4 — Vite SPA [ ] +- [ ] Vite project scaffold (package.json, vite.config, index.html) +- [ ] API client w/ credentials:'include' +- [ ] Feed → detail → auth → bookmarks → settings → admin +- [ ] Flask serves built dist/ with catch-all fallback; dev proxy → Flask + +## Phase 5 — Cut over (gated on SPA parity) [-] +Deferred: deleting the Jinja render path is destructive and only safe once the +SPA reaches parity AND can be runtime-verified. Will not delete templates this +pass. Phase 5 also switches Dockerfile off `flask run` to a real WSGI server. + +## Phase 6 — Hardening [~] +- [x] pytest scaffold (services + /api/v1 contracts) — `pytest.ini`, `tests/conftest.py` + (in-memory SQLite app fixture, no Postgres; stubbed polling/filter singletons), + `tests/test_app_factory.py` (endpoint registration + no module-level `app`), + `tests/test_api_contracts.py` (posts/post_detail/comments/filters JSON shape with + monkeypatched `post_service` + `get_filter_engine`), `tests/test_filter_pipeline.py` + (offline: registry discovery, offline plugin filterset, AI-disabled fail-open, + comment tree modes), `tests/test_plugin_contract.py` (drop-in stage/plugin) +- [ ] ruff config + CI workflow +- [ ] charset/encoding CI gate +- [x] plugin contract test (drop-in stage/plugin discovered with zero core edits) — + `tests/test_plugin_contract.py` registers a stage + plugin only in the test + module via the public decorators and asserts the engine instantiates and runs + it, plus a throwaway on-disk config selecting it + +Verification note: `tests/test_filter_pipeline.py` and `tests/test_plugin_contract.py` +are Flask-free and were exercised locally with a plain-python harness (12/12 pass) +since `pytest` is not installed locally. The app-factory / API-contract tests +`py_compile` clean and run in CI/docker where Flask/SQLAlchemy/bcrypt are present. + +## Verification constraints +No venv/Flask deps or Postgres available locally (see memory `env-no-local-runtime`). +All Python verified via `python -m py_compile` only; runtime/endpoint checks +require `docker compose up` or a deps-installed venv. SPA build needs npm. \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..46328f1 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,8 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_classes = Test* +python_functions = test_* +addopts = -ra -q +filterwarnings = + ignore::DeprecationWarning \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2555dd4..3a2b76a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,5 @@ authlib==1.3.2 APScheduler==3.10.4 praw==7.7.1 feedparser==6.0.12 +# Test harness (Agent E) +pytest==8.3.4 diff --git a/routes/__init__.py b/routes/__init__.py new file mode 100644 index 0000000..ee920f5 --- /dev/null +++ b/routes/__init__.py @@ -0,0 +1 @@ +"""Route registration modules.""" diff --git a/routes/admin.py b/routes/admin.py new file mode 100644 index 0000000..1d2653b --- /dev/null +++ b/routes/admin.py @@ -0,0 +1,304 @@ +"""Admin route registration.""" + +import logging +import os +import shutil +from datetime import datetime + +from flask import flash, redirect, render_template, request, url_for +from flask_login import current_user, login_required + +from database import db +from extensions import get_polling_service +from models import PollLog, PollSource +from services import load_platform_config, post_service + +logger = logging.getLogger(__name__) + + +def _require_admin(redirect_endpoint="index", message="Access denied"): + if current_user.is_admin: + return None + flash(message, "error") + return redirect(url_for(redirect_endpoint)) + + +def register_admin_routes(app, user_service): + """Register admin routes while preserving endpoint names.""" + + @app.route("/admin") + @login_required + def admin_panel(): + """Admin panel - user management.""" + denied = _require_admin("index", "Access denied. Admin privileges required.") + if denied: + return denied + + if not user_service: + flash("User service not available", "error") + return redirect(url_for("index")) + + users = user_service.get_all_users() + return render_template("admin.html", users=users) + + @app.route("/admin/user//delete", methods=["POST"]) + @login_required + def admin_delete_user(user_id): + """Delete user (admin only).""" + denied = _require_admin("index") + if denied: + return denied + + if current_user.id == user_id: + flash("You cannot delete your own account!", "error") + return redirect(url_for("admin_panel")) + + user = user_service.get_user_by_id(user_id) + if user: + username = user.username + if user_service.delete_user(user_id): + flash(f"User {username} has been deleted.", "success") + logger.info(f"Admin {current_user.id} deleted user {username} ({user_id})") + else: + flash("Error deleting user", "error") + logger.error(f"Failed to delete user {user_id}") + else: + flash("User not found", "error") + + return redirect(url_for("admin_panel")) + + @app.route("/admin/user//toggle-admin", methods=["POST"]) + @login_required + def admin_toggle_admin(user_id): + """Toggle user admin status.""" + denied = _require_admin("index") + if denied: + return denied + + target_user = user_service.get_user_by_id(user_id) + if target_user: + user_service.update_user_admin_status(user_id, not target_user.is_admin) + flash("Admin status updated", "success") + else: + flash("User not found", "error") + + return redirect(url_for("admin_panel")) + + @app.route("/admin/clear_cache", methods=["POST"]) + @login_required + def admin_clear_cache(): + """Clear application cache.""" + denied = _require_admin("admin_panel") + if denied: + return denied + + try: + for cache_dir in ["cache", "temp"]: + if os.path.exists(cache_dir): + shutil.rmtree(cache_dir) + post_service.invalidate() + flash("Cache cleared successfully", "success") + logger.info(f"Cache cleared by admin user {current_user.id}") + except Exception as e: + flash(f"Error clearing cache: {str(e)}", "error") + logger.error(f"Cache clearing error: {e}") + + return redirect(url_for("admin_panel")) + + @app.route("/admin/backup_data", methods=["POST"]) + @login_required + def admin_backup_data(): + """Create backup of application data.""" + denied = _require_admin("admin_panel") + if denied: + return denied + + try: + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + backup_name = f"balanceboard_backup_{timestamp}" + backup_dir = f"backups/{backup_name}" + os.makedirs(backup_dir, exist_ok=True) + + for dir_name in ["data", "templates", "themes", "static"]: + if os.path.exists(dir_name): + shutil.copytree(dir_name, f"{backup_dir}/{dir_name}") + + for file_name in ["app.py", "models.py", "database.py", "filtersets.json"]: + if os.path.exists(file_name): + shutil.copy2(file_name, backup_dir) + + flash(f"Backup created: {backup_name}", "success") + except Exception as e: + flash(f"Error creating backup: {str(e)}", "error") + + return redirect(url_for("admin_panel")) + + @app.route("/admin/polling") + @login_required + def admin_polling(): + """Admin polling management page.""" + denied = _require_admin("index", "Access denied. Admin privileges required.") + if denied: + return denied + + sources = PollSource.query.order_by(PollSource.platform, PollSource.display_name).all() + scheduler_status = get_polling_service().get_status() + platform_config = load_platform_config() + return render_template( + "admin_polling.html", + sources=sources, + scheduler_status=scheduler_status, + platform_config=platform_config, + ) + + @app.route("/admin/polling/add", methods=["POST"]) + @login_required + def admin_polling_add(): + """Add a new poll source.""" + denied = _require_admin("index") + if denied: + return denied + + platform = request.form.get("platform") + source_id = request.form.get("source_id") + custom_source_id = request.form.get("custom_source_id") + display_name = request.form.get("display_name") + poll_interval = int(request.form.get("poll_interval", 60)) + max_posts = int(request.form.get("max_posts", 100)) + fetch_comments = request.form.get("fetch_comments", "true") == "true" + priority = request.form.get("priority", "medium") + + if custom_source_id and custom_source_id.strip(): + source_id = custom_source_id.strip() + + if not platform or not source_id or not display_name: + flash("Missing required fields", "error") + return redirect(url_for("admin_polling")) + + existing = PollSource.query.filter_by(platform=platform, source_id=source_id).first() + if existing: + flash(f"Source {platform}:{source_id} already exists", "warning") + return redirect(url_for("admin_polling")) + + source = PollSource( + platform=platform, + source_id=source_id, + display_name=display_name, + poll_interval_minutes=poll_interval, + max_posts=max_posts, + fetch_comments=fetch_comments, + priority=priority, + enabled=False, + created_by=current_user.id, + ) + db.session.add(source) + db.session.commit() + + flash(f"Added polling source: {display_name}", "success") + logger.info(f"Admin {current_user.id} added poll source {platform}:{source_id}") + return redirect(url_for("admin_polling")) + + @app.route("/admin/polling//toggle", methods=["POST"]) + @login_required + def admin_polling_toggle(source_id): + """Toggle a poll source on/off.""" + denied = _require_admin("index") + if denied: + return denied + + source = PollSource.query.get(source_id) + if not source: + flash("Source not found", "error") + return redirect(url_for("admin_polling")) + + source.enabled = not source.enabled + db.session.commit() + status = "enabled" if source.enabled else "disabled" + flash(f"Polling {status} for {source.display_name}", "success") + return redirect(url_for("admin_polling")) + + @app.route("/admin/polling//update", methods=["POST"]) + @login_required + def admin_polling_update(source_id): + """Update poll source configuration.""" + denied = _require_admin("index") + if denied: + return denied + + source = PollSource.query.get(source_id) + if not source: + flash("Source not found", "error") + return redirect(url_for("admin_polling")) + + if request.form.get("poll_interval"): + source.poll_interval_minutes = int(request.form.get("poll_interval")) + if request.form.get("max_posts"): + source.max_posts = int(request.form.get("max_posts")) + if request.form.get("fetch_comments") is not None: + source.fetch_comments = request.form.get("fetch_comments") == "true" + if request.form.get("priority"): + source.priority = request.form.get("priority") + if request.form.get("display_name"): + source.display_name = request.form.get("display_name") + + db.session.commit() + flash(f"Updated settings for {source.display_name}", "success") + return redirect(url_for("admin_polling")) + + @app.route("/admin/polling//poll-now", methods=["POST"]) + @login_required + def admin_polling_poll_now(source_id): + """Manually trigger polling for a source.""" + denied = _require_admin("index") + if denied: + return denied + + source = PollSource.query.get(source_id) + if not source: + flash("Source not found", "error") + return redirect(url_for("admin_polling")) + + try: + get_polling_service().poll_now(source_id) + flash(f"Polling started for {source.display_name}", "success") + except Exception as e: + flash(f"Error starting poll: {str(e)}", "error") + logger.error(f"Error triggering poll for {source_id}: {e}") + + return redirect(url_for("admin_polling")) + + @app.route("/admin/polling//delete", methods=["POST"]) + @login_required + def admin_polling_delete(source_id): + """Delete a poll source.""" + denied = _require_admin("index") + if denied: + return denied + + source = PollSource.query.get(source_id) + if not source: + flash("Source not found", "error") + return redirect(url_for("admin_polling")) + + display_name = source.display_name + db.session.delete(source) + db.session.commit() + flash(f"Deleted polling source: {display_name}", "success") + logger.info(f"Admin {current_user.id} deleted poll source {source_id}") + return redirect(url_for("admin_polling")) + + @app.route("/admin/polling//logs") + @login_required + def admin_polling_logs(source_id): + """View logs for a specific poll source.""" + denied = _require_admin("index") + if denied: + return denied + + source = PollSource.query.get(source_id) + if not source: + flash("Source not found", "error") + return redirect(url_for("admin_polling")) + + logs = source.logs.limit(50).all() + return render_template("admin_polling_logs.html", source=source, logs=logs) \ No newline at end of file diff --git a/routes/assets.py b/routes/assets.py new file mode 100644 index 0000000..68b8e9c --- /dev/null +++ b/routes/assets.py @@ -0,0 +1,41 @@ +"""Asset and static-file route registration.""" + +import logging +import os + +from flask import abort, current_app, send_from_directory + +from security import is_safe_path + +logger = logging.getLogger(__name__) + + +def register_asset_routes(app): + """Register asset routes while preserving legacy endpoint names.""" + + @app.route("/themes/") + def serve_theme(filename): + """Serve theme files (CSS, JS).""" + if not is_safe_path(filename) or ".." in filename: + logger.warning(f"Unsafe theme file requested: {filename}") + abort(404) + return send_from_directory("themes", filename) + + @app.route("/logo.png") + def serve_logo(): + """Serve configurable logo.""" + logo_path = current_app.config["LOGO_PATH"] + if "/" not in logo_path: + return send_from_directory(".", logo_path) + + directory = os.path.dirname(logo_path) + filename = os.path.basename(logo_path) + return send_from_directory(directory, filename) + + @app.route("/static/") + def serve_static(filename): + """Serve static files (avatars, etc.).""" + if not is_safe_path(filename) or ".." in filename: + logger.warning(f"Unsafe static file requested: {filename}") + abort(404) + return send_from_directory("static", filename) \ No newline at end of file diff --git a/routes/auth.py b/routes/auth.py new file mode 100644 index 0000000..a6811e9 --- /dev/null +++ b/routes/auth.py @@ -0,0 +1,290 @@ +"""Authentication route registration.""" + +import logging +from urllib.parse import quote_plus, urlencode + +from flask import current_app, flash, redirect, render_template, request, session, url_for +from flask_login import current_user, login_required, login_user, logout_user + +from config import MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH +from models import User + +logger = logging.getLogger(__name__) + + +def register_auth_routes(app, user_service, auth0_client): + """Register auth routes while preserving legacy endpoint names.""" + + @app.route("/login", methods=["GET", "POST"]) + def login(): + """Login page.""" + if current_user.is_authenticated: + return redirect(url_for("index")) + + auth0_configured = bool( + current_app.config.get("AUTH0_DOMAIN") + and current_app.config.get("AUTH0_CLIENT_ID") + ) + + if request.method == "POST": + username = request.form.get("username") + password = request.form.get("password") + remember = request.form.get("remember", False) == "on" + + if not user_service: + flash("User service not available", "error") + return render_template("login.html", auth0_configured=auth0_configured) + + user = user_service.authenticate(username, password) + if user: + login_user(user, remember=remember) + flash(f"Welcome back, {user.username}!", "success") + next_page = request.args.get("next") + return redirect(next_page) if next_page else redirect(url_for("index")) + + flash("Invalid username or password", "error") + + return render_template("login.html", auth0_configured=auth0_configured) + + @app.route("/password-reset-request", methods=["GET", "POST"]) + def password_reset_request(): + """Request a password reset.""" + if current_user.is_authenticated: + return redirect(url_for("index")) + + if request.method == "POST": + email = request.form.get("email", "").strip().lower() + if not email: + flash("Please enter your email address", "error") + return render_template("password_reset_request.html") + + user = User.query.filter_by(email=email).first() + flash( + "If an account exists with that email, a password reset link has been sent.", + "success", + ) + + if user and user.password_hash: + token = user.generate_reset_token() + reset_url = url_for("password_reset", token=token, _external=True) + logger.info(f"Password reset requested for {email}. Reset URL: {reset_url}") + flash(f"Reset link (development only): {reset_url}", "info") + + return redirect(url_for("login")) + + return render_template("password_reset_request.html") + + @app.route("/password-reset/", methods=["GET", "POST"]) + def password_reset(token): + """Reset password with token.""" + if current_user.is_authenticated: + return redirect(url_for("index")) + + user = User.query.filter_by(reset_token=token).first() + if not user or not user.verify_reset_token(token): + flash("Invalid or expired reset token", "error") + return redirect(url_for("login")) + + if request.method == "POST": + password = request.form.get("password", "") + confirm_password = request.form.get("confirm_password", "") + + if not password or len(password) < MIN_PASSWORD_LENGTH: + flash(f"Password must be at least {MIN_PASSWORD_LENGTH} characters", "error") + return render_template("password_reset.html") + + if password != confirm_password: + flash("Passwords do not match", "error") + return render_template("password_reset.html") + + user.set_password(password) + user.clear_reset_token() + flash("Your password has been reset successfully. You can now log in.", "success") + return redirect(url_for("login")) + + return render_template("password_reset.html") + + @app.route("/auth0/login") + def auth0_login(): + """Redirect to Auth0 for authentication.""" + if not current_app.config.get("AUTH0_DOMAIN") or not current_app.config.get("AUTH0_CLIENT_ID"): + flash( + "Auth0 authentication is not configured. Please use email/password login or contact the administrator.", + "error", + ) + return redirect(url_for("login")) + + try: + redirect_uri = url_for("auth0_callback", _external=True) + return auth0_client.authorize_redirect(redirect_uri) + except Exception as e: + logger.error(f"Auth0 login error: {e}") + flash("Auth0 authentication failed. Please use email/password login.", "error") + return redirect(url_for("login")) + + @app.route("/auth0/callback") + def auth0_callback(): + """Handle Auth0 callback and create/login user.""" + try: + token = auth0_client.authorize_access_token() + user_info = token.get("userinfo") + if not user_info: + user_info = auth0_client.parse_id_token(token) + + auth0_id = user_info.get("sub") + email = user_info.get("email") + username = ( + user_info.get("nickname") + or user_info.get("preferred_username") + or email.split("@")[0] + ) + + if not auth0_id or not email: + flash("Unable to get user information from Auth0", "error") + return redirect(url_for("login")) + + user = user_service.get_user_by_auth0_id(auth0_id) + if not user: + existing_user = user_service.get_user_by_email(email) + if existing_user: + user_service.link_auth0_account(existing_user.id, auth0_id) + user = existing_user + flash(f"Account linked successfully! Welcome back, {user.username}!", "success") + else: + base_username = username[:MAX_USERNAME_LENGTH - 3] + unique_username = base_username + counter = 1 + while user_service.username_exists(unique_username): + unique_username = f"{base_username}_{counter}" + counter += 1 + + user_id = user_service.create_user( + username=unique_username, + email=email, + password=None, + is_admin=False, + auth0_id=auth0_id, + ) + if user_id: + user = user_service.get_user_by_id(user_id) + flash(f"Account created successfully! Welcome, {user.username}!", "success") + else: + flash("Failed to create user account", "error") + return redirect(url_for("login")) + else: + flash(f"Welcome back, {user.username}!", "success") + + if user: + login_user(user, remember=True) + session["auth0_user_info"] = user_info + next_page = request.args.get("next") + return redirect(next_page) if next_page else redirect(url_for("index")) + except Exception as e: + logger.error(f"Auth0 callback error: {e}") + flash("Authentication failed. Please try again.", "error") + + return redirect(url_for("login")) + + @app.route("/auth0/logout") + @login_required + def auth0_logout(): + """Logout from Auth0 and local session.""" + session.clear() + logout_user() + + domain = current_app.config["AUTH0_DOMAIN"] + client_id = current_app.config["AUTH0_CLIENT_ID"] + return_to = url_for("index", _external=True) + logout_url = f"https://{domain}/v2/logout?" + urlencode( + {"returnTo": return_to, "client_id": client_id}, quote_via=quote_plus + ) + return redirect(logout_url) + + @app.route("/admin-setup", methods=["GET", "POST"]) + def admin_setup(): + """Create first admin user.""" + try: + user_count = User.query.count() + if user_count > 0: + flash("Admin user already exists.", "info") + return redirect(url_for("login")) + except Exception as e: + logger.warning(f"Database error checking existing users: {e}") + + if request.method == "POST": + username = request.form.get("username") + email = request.form.get("email") + password = request.form.get("password") + password_confirm = request.form.get("password_confirm") + + if not username or not email or not password: + flash("All fields are required", "error") + return render_template("admin_setup.html") + + if password != password_confirm: + flash("Passwords do not match", "error") + return render_template("admin_setup.html") + + if len(password) < MIN_PASSWORD_LENGTH: + flash(f"Password must be at least {MIN_PASSWORD_LENGTH} characters", "error") + return render_template("admin_setup.html") + + user_id = user_service.create_user(username, email, password, is_admin=True) + if user_id: + flash("Admin account created successfully! Please log in.", "success") + return redirect(url_for("login")) + flash("Error creating admin account. Please try again.", "error") + + return render_template("admin_setup.html") + + @app.route("/signup", methods=["GET", "POST"]) + def signup(): + """Signup page.""" + if current_user.is_authenticated: + return redirect(url_for("index")) + + if request.method == "POST": + username = request.form.get("username") + email = request.form.get("email") + password = request.form.get("password") + password_confirm = request.form.get("password_confirm") + + if not user_service: + flash("User service not available", "error") + return render_template("signup.html") + + if not username or not email or not password: + flash("All fields are required", "error") + return render_template("signup.html") + + if password != password_confirm: + flash("Passwords do not match", "error") + return render_template("signup.html") + + if len(password) < MIN_PASSWORD_LENGTH: + flash(f"Password must be at least {MIN_PASSWORD_LENGTH} characters", "error") + return render_template("signup.html") + + if user_service.username_exists(username): + flash("Username already taken", "error") + return render_template("signup.html") + + if user_service.email_exists(email): + flash("Email already registered", "error") + return render_template("signup.html") + + user_id = user_service.create_user(username, email, password) + if user_id: + flash("Account created successfully! Please log in.", "success") + return redirect(url_for("login")) + flash("Error creating account. Please try again.", "error") + + return render_template("signup.html") + + @app.route("/logout") + @login_required + def logout(): + """Logout current user.""" + logout_user() + flash("You have been logged out.", "info") + return redirect(url_for("index")) \ No newline at end of file diff --git a/routes/pages.py b/routes/pages.py new file mode 100644 index 0000000..2f4015e --- /dev/null +++ b/routes/pages.py @@ -0,0 +1,96 @@ +"""Page route registration for the legacy Jinja UI.""" + +import json +import logging + +from flask import current_app, redirect, render_template, url_for +from flask_login import current_user, login_required + +from services import get_display_name_for_source, load_platform_config, post_service + +logger = logging.getLogger(__name__) + + +def _load_user_settings(): + if not current_user.is_authenticated: + return {} + try: + return json.loads(current_user.settings) if current_user.settings else {} + except (json.JSONDecodeError, TypeError): + return {} + + +def register_page_routes(app): + """Register legacy page routes while preserving endpoint names.""" + + @app.route("/") + def index(): + """Serve the main feed page.""" + quick_stats = post_service.quick_stats() + + if current_user.is_authenticated: + return render_template( + "dashboard.html", + user_settings=_load_user_settings(), + quick_stats=quick_stats, + ) + + if current_app.config.get("ALLOW_ANONYMOUS_ACCESS", False): + user_settings = { + "filter_set": "no_filter", + "communities": [], + "experience": { + "infinite_scroll": False, + "auto_refresh": False, + "push_notifications": False, + "dark_patterns_opt_in": False, + "time_filter_enabled": False, + "time_filter_days": 7, + }, + } + return render_template( + "dashboard.html", + user_settings=user_settings, + anonymous=True, + quick_stats=quick_stats, + ) + + return redirect(url_for("login")) + + @app.route("/bookmarks") + @login_required + def bookmarks(): + """Bookmarks page.""" + return render_template("bookmarks.html", user=current_user) + + @app.route("/post/") + def post_detail(post_id): + """Serve individual post detail page with modern theme.""" + try: + platform_config = load_platform_config() + cached_posts, cached_comments = post_service.load() + + post_data = cached_posts.get(post_id) + if not post_data: + return render_template("404.html"), 404 + + post = dict(post_data) + post["source_display"] = get_display_name_for_source( + post.get("platform", ""), + post.get("source", ""), + platform_config, + ) + + comments_flat = cached_comments.get(post_id, []) + logger.info(f"Loading post {post_id}: found {len(comments_flat)} comments") + comments = post_service.build_comment_tree(comments_flat) + + return render_template( + "post_detail.html", + post=post, + comments=comments, + user_settings=_load_user_settings(), + ) + except Exception as e: + logger.error(f"Error loading post {post_id}: {e}") + return render_template("404.html"), 404 \ No newline at end of file diff --git a/routes/settings.py b/routes/settings.py new file mode 100644 index 0000000..a83b817 --- /dev/null +++ b/routes/settings.py @@ -0,0 +1,310 @@ +"""Settings and profile route registration.""" + +import json +import logging +import os + +from flask import current_app, flash, redirect, render_template, request, url_for +from flask_login import current_user, login_required +from werkzeug.utils import secure_filename + +from config import MAX_FILENAME_LENGTH, UPLOAD_FOLDER +from database import db +from extensions import get_filter_engine +from security import is_allowed_file, is_safe_filterset +from services import SettingsService, load_platform_config + +logger = logging.getLogger(__name__) + + +DEFAULT_AVATARS = [ + {"id": "default_1", "name": "Gradient Blue", "bg": "linear-gradient(135deg, #667eea 0%, #764ba2 100%)"}, + {"id": "default_2", "name": "Gradient Green", "bg": "linear-gradient(135deg, #4facfe 0%, #00f2fe 100%)"}, + {"id": "default_3", "name": "Gradient Orange", "bg": "linear-gradient(135deg, #fa709a 0%, #fee140 100%)"}, + {"id": "default_4", "name": "Gradient Purple", "bg": "linear-gradient(135deg, #a8edea 0%, #fed6e3 100%)"}, + {"id": "default_5", "name": "Brand Colors", "bg": "linear-gradient(135deg, #4db6ac 0%, #26a69a 100%)"}, + {"id": "default_6", "name": "Sunset", "bg": "linear-gradient(135deg, #ff7e5f 0%, #feb47b 100%)"}, +] + + +def register_settings_routes(app, user_service): + """Register settings/profile routes while preserving endpoint names.""" + + @app.route("/settings") + @login_required + def settings(): + """Main settings page.""" + user_settings = SettingsService.parse(current_user.settings) + + try: + with open("filtersets.json", "r", encoding="utf-8") as f: + filter_sets = json.load(f) + except (FileNotFoundError, json.JSONDecodeError, IOError): + filter_sets = {} + + return render_template( + "settings.html", + user=current_user, + user_settings=user_settings, + filter_sets=filter_sets, + ) + + @app.route("/settings/profile", methods=["GET", "POST"]) + @login_required + def settings_profile(): + """Profile settings page.""" + if request.method == "POST": + username = request.form.get("username") + email = request.form.get("email") + default_avatar = request.form.get("default_avatar") + + if not username or not email: + flash("Username and email are required", "error") + return render_template("settings_profile.html", user=current_user) + + if username != current_user.username and user_service.username_exists(username): + flash("Username already taken", "error") + return render_template("settings_profile.html", user=current_user) + + if email != current_user.email and user_service.email_exists(email): + flash("Email already registered", "error") + return render_template("settings_profile.html", user=current_user) + + current_user.username = username + current_user.email = email + + if default_avatar and default_avatar.startswith("default_"): + current_user.profile_picture_url = f"/static/default-avatars/{default_avatar}.png" + + db.session.commit() + flash("Profile updated successfully", "success") + return redirect(url_for("settings")) + + return render_template( + "settings_profile.html", + user=current_user, + default_avatars=DEFAULT_AVATARS, + ) + + @app.route("/settings/communities", methods=["GET", "POST"]) + @login_required + def settings_communities(): + """Community/source selection settings.""" + if request.method == "POST": + user_settings = SettingsService.parse(current_user.settings) + user_settings["communities"] = request.form.getlist("communities") + current_user.settings = json.dumps(user_settings) + db.session.commit() + flash("Community preferences updated", "success") + return redirect(url_for("settings")) + + user_settings = SettingsService.parse(current_user.settings) + selected_communities = user_settings.get("communities", []) + available_communities = [] + + try: + platform_config = load_platform_config() or {"platforms": {}, "collection_targets": []} + except Exception as e: + logger.error(f"Error loading platform config: {e}") + platform_config = {"platforms": {}, "collection_targets": []} + + enabled_communities = set() + try: + for target in platform_config.get("collection_targets", []): + if "platform" in target and "community" in target: + enabled_communities.add((target["platform"], target["community"])) + except Exception as e: + logger.error(f"Error processing collection_targets: {e}") + + try: + for platform_name, platform_info in platform_config.get("platforms", {}).items(): + if not isinstance(platform_info, dict): + continue + communities = platform_info.get("communities", []) + if not isinstance(communities, list): + continue + + for community_info in communities: + try: + if not isinstance(community_info, dict): + continue + if (platform_name, community_info["id"]) in enabled_communities: + available_communities.append( + { + "id": community_info["id"], + "name": community_info["name"], + "display_name": community_info.get("display_name", community_info["name"]), + "platform": platform_name, + "icon": community_info.get("icon", platform_info.get("icon", "\U0001f4c4")), + "description": community_info.get("description", ""), + } + ) + except Exception as e: + logger.error(f"Error processing community {community_info}: {e}") + continue + except Exception as e: + logger.error(f"Error building community list: {e}") + + logger.info(f"Found {len(available_communities)} available communities") + return render_template( + "settings_communities.html", + user=current_user, + available_communities=available_communities, + selected_communities=selected_communities, + ) + + @app.route("/settings/filters", methods=["GET", "POST"]) + @login_required + def settings_filters(): + """Filter settings page.""" + if request.method == "POST": + selected_filter = request.form.get("filter_set", "no_filter") + user_settings = SettingsService.validate(current_user.settings) + + if is_safe_filterset(selected_filter): + user_settings["filter_set"] = selected_filter + else: + flash("Invalid filter selection", "error") + return redirect(url_for("settings")) + + try: + current_user.settings = json.dumps(user_settings) + db.session.commit() + flash("Filter settings updated successfully", "success") + except Exception as e: + db.session.rollback() + logger.error(f"Error saving filter settings for user {current_user.id}: {e}") + flash("Error saving settings", "error") + + return redirect(url_for("settings")) + + user_settings = SettingsService.parse(current_user.settings) + current_filter = user_settings.get("filter_set", "no_filter") + filter_engine = get_filter_engine() + filter_sets = { + filterset_name: filter_engine.config.get_filterset(filterset_name) + for filterset_name in filter_engine.get_available_filtersets() + } + + return render_template( + "settings_filters.html", + user=current_user, + filter_sets=filter_sets, + current_filter=current_filter, + ) + + @app.route("/settings/experience", methods=["GET", "POST"]) + @login_required + def settings_experience(): + """Experience and behavioral settings page.""" + if request.method == "POST": + user_settings = SettingsService.parse(current_user.settings) + user_settings["experience"] = { + "infinite_scroll": request.form.get("infinite_scroll") == "on", + "auto_refresh": request.form.get("auto_refresh") == "on", + "push_notifications": request.form.get("push_notifications") == "on", + "dark_patterns_opt_in": request.form.get("dark_patterns_opt_in") == "on", + "time_filter_enabled": request.form.get("time_filter_enabled") == "on", + "time_filter_days": int(request.form.get("time_filter_days", 7)), + } + current_user.settings = json.dumps(user_settings) + db.session.commit() + flash("Experience settings updated successfully", "success") + return redirect(url_for("settings")) + + return render_template( + "settings_experience.html", + user=current_user, + experience_settings=SettingsService.experience_settings(current_user.settings), + ) + + @app.route("/upload-avatar", methods=["POST"]) + @login_required + def upload_avatar(): + """Upload profile picture.""" + try: + logger.info(f"Avatar upload attempt by user {current_user.id} ({current_user.username})") + logger.debug(f"Request files: {list(request.files.keys())}") + logger.debug(f"Request form: {dict(request.form)}") + + if not hasattr(current_user, "id") or not current_user.id: + logger.error("User missing ID attribute") + flash("Authentication error. Please log in again.", "error") + return redirect(url_for("login")) + + if not hasattr(current_user, "username") or not current_user.username: + logger.error("User missing username attribute") + flash("User profile incomplete. Please update your profile.", "error") + return redirect(url_for("settings_profile")) + + if "avatar" not in request.files: + logger.warning("No avatar file in request") + flash("No file selected", "error") + return redirect(url_for("settings_profile")) + + file = request.files["avatar"] + if file.filename == "": + logger.warning("Empty filename provided") + flash("No file selected", "error") + return redirect(url_for("settings_profile")) + + logger.info(f"Processing file: {file.filename}") + if not is_allowed_file(file.filename): + logger.warning(f"Invalid file type: {file.filename}") + flash("Invalid file type. Please upload PNG, JPG, or GIF", "error") + return redirect(url_for("settings_profile")) + + max_content_length = current_app.config.get("MAX_CONTENT_LENGTH", 16 * 1024 * 1024) + if hasattr(file, "content_length") and file.content_length > max_content_length: + logger.warning(f"File too large: {file.content_length}") + flash("File too large. Maximum size is 16MB", "error") + return redirect(url_for("settings_profile")) + + filename = secure_filename(file.filename) + if not filename or len(filename) > MAX_FILENAME_LENGTH: + logger.warning(f"Invalid filename after sanitization: {filename}") + flash("Invalid filename", "error") + return redirect(url_for("settings_profile")) + + unique_filename = f"{current_user.id}_{filename}" + logger.info(f"Generated unique filename: {unique_filename}") + + upload_dir = os.path.abspath(UPLOAD_FOLDER) + os.makedirs(upload_dir, exist_ok=True) + upload_path = os.path.join(upload_dir, unique_filename) + + if not os.path.abspath(upload_path).startswith(upload_dir): + logger.warning(f"Path traversal attempt in file upload: {upload_path}") + flash("Invalid file path", "error") + return redirect(url_for("settings_profile")) + + file.save(upload_path) + logger.info(f"File saved successfully: {upload_path}") + + old_avatar_url = current_user.profile_picture_url + current_user.profile_picture_url = f"/static/avatars/{unique_filename}" + db.session.commit() + logger.info(f"User profile updated successfully for {current_user.username}") + + if old_avatar_url and old_avatar_url.startswith("/static/avatars/") and current_user.id in old_avatar_url: + try: + old_file_path = os.path.join(upload_dir, os.path.basename(old_avatar_url)) + if os.path.exists(old_file_path): + os.remove(old_file_path) + logger.info(f"Cleaned up old avatar: {old_file_path}") + except Exception as e: + logger.warning(f"Could not clean up old avatar: {e}") + + flash("Profile picture updated successfully", "success") + return redirect(url_for("settings_profile")) + except Exception as e: + logger.error(f"Unexpected error in avatar upload: {e}") + db.session.rollback() + flash("An unexpected error occurred. Please try again.", "error") + return redirect(url_for("settings_profile")) + + @app.route("/profile") + @login_required + def profile(): + """User profile page.""" + return render_template("profile.html", user=current_user) \ No newline at end of file diff --git a/run_app.py b/run_app.py index 773f569..6725252 100755 --- a/run_app.py +++ b/run_app.py @@ -6,7 +6,7 @@ Starts the Flask web app with PostgreSQL/SQLAlchemy integration. import os import sys -from app import app +from app import create_app def main(): @@ -52,6 +52,7 @@ def main(): # Run Flask app debug_mode = os.getenv('FLASK_DEBUG', 'True').lower() == 'true' + app = create_app() app.run(host=host, port=port, debug=debug_mode) diff --git a/security.py b/security.py new file mode 100644 index 0000000..468af96 --- /dev/null +++ b/security.py @@ -0,0 +1,37 @@ +"""Security helper functions shared across route modules.""" + +import re + +from config import ALLOWED_EXTENSIONS +from extensions import get_filter_engine + + +def is_safe_filterset(filterset): + """Validate a filterset name against the filter engine's known filtersets + and a safe-character whitelist. Fails closed if the filter engine has not + been initialized yet. + """ + if not filterset or not isinstance(filterset, str): + return False + try: + allowed = set(get_filter_engine().get_available_filtersets()) + except Exception: + return False + return filterset in allowed and re.match(r"^[a-zA-Z0-9_-]+$", filterset) is not None + + +def is_safe_path(path): + """Validate a relative file path against directory-traversal attempts.""" + if not path or not isinstance(path, str): + return False + if ".." in path or path.startswith("/") or "\\" in path: + return False + return re.match(r"^[a-zA-Z0-9._/-]+$", path) is not None + + +def is_allowed_file(filename): + """Check whether an uploaded filename has an allowed image extension.""" + return ( + "." in filename + and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS + ) \ No newline at end of file diff --git a/services/__init__.py b/services/__init__.py new file mode 100644 index 0000000..6e070df --- /dev/null +++ b/services/__init__.py @@ -0,0 +1,17 @@ +"""Service layer for BalanceBoard. + +Services hold business logic shared across route modules, keeping the route +handlers thin. Posts/comments still come from disk JSON in this phase (moved +to Postgres in Phase 3); ``PostService`` owns that cache. +""" + +from .post_service import PostService, load_platform_config, get_display_name_for_source, post_service +from .settings_service import SettingsService + +__all__ = [ + "PostService", + "post_service", + "load_platform_config", + "get_display_name_for_source", + "SettingsService", +] \ No newline at end of file diff --git a/services/post_service.py b/services/post_service.py new file mode 100644 index 0000000..011fdad --- /dev/null +++ b/services/post_service.py @@ -0,0 +1,252 @@ +"""Post/comment data service. + +Owns the short-lived in-memory cache of posts and comments. Phase 3 makes +Postgres the primary source of truth; the legacy ``data/*.json`` reader remains +as a fallback for local/dev environments before the backfill has run. +""" + +import json +import logging +import time +from collections import defaultdict +from datetime import datetime, timedelta +from pathlib import Path + +from sqlalchemy import func + +from models import Comment, Post + +logger = logging.getLogger(__name__) + +_CACHE_DURATION = 300 # 5 minutes + + +def load_platform_config(): + """Load platform configuration from ``platform_config.json``. + + Returns a safe default (empty platforms, no targets) on any error so + callers can iterate without extra guarding. + """ + try: + with open("platform_config.json", "r", encoding="utf-8") as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError, IOError) as e: + logger.warning(f"Could not load platform config: {e}") + return {"platforms": {}, "collection_targets": []} + + +def get_display_name_for_source(platform, source, platform_config): + """Get a human display name for a (platform, source) pair.""" + if not platform_config or "platforms" not in platform_config: + return source + + platform_info = platform_config["platforms"].get(platform, {}) + + if platform_info.get("supports_communities"): + for community in platform_info.get("communities", []): + if community["id"] == source: + return community["display_name"] + prefix = platform_info.get("prefix", "") + return f"{prefix}{source}" if source else platform_info.get("name", platform) + return platform_info.get("name", platform) + + +class PostService: + """Cache and serve posts/comments from Postgres, plus derived views.""" + + def __init__(self, cache_duration=_CACHE_DURATION): + self.post_cache = {} + self.comment_cache = defaultdict(list) + self.cache_timestamp = 0 + self.cache_duration = cache_duration + self.cache_source = None + + def load(self): + """Return (post_cache, comment_cache), refreshing from Postgres if stale.""" + current_time = time.time() + if current_time - self.cache_timestamp < self.cache_duration and self.post_cache: + return self.post_cache, self.comment_cache + + self.post_cache.clear() + self.comment_cache.clear() + + loaded_from_db = self._load_from_db() + if not loaded_from_db: + self._load_from_disk() + self.cache_source = "disk" + else: + self.cache_source = "db" + + self.cache_timestamp = current_time + logger.info( + f"Cache refreshed from {self.cache_source}: {len(self.post_cache)} posts, " + f"{len(self.comment_cache)} comment groups" + ) + return self.post_cache, self.comment_cache + + def _load_from_db(self): + """Populate caches from Postgres. Return False if unavailable or empty.""" + try: + posts = Post.query.order_by(Post.timestamp.desc()).all() + if not posts: + return False + + for post in posts: + self.post_cache[post.uuid] = self._post_to_dict(post) + + comments = Comment.query.order_by(Comment.timestamp.asc()).all() + for comment in comments: + self.comment_cache[comment.post_uuid].append(self._comment_to_dict(comment)) + return True + except Exception as e: + logger.warning(f"Postgres content load unavailable; falling back to disk: {e}") + self.post_cache.clear() + self.comment_cache.clear() + return False + + def _load_from_disk(self): + posts_dir = Path("data/posts") + comments_dir = Path("data/comments") + + if posts_dir.exists(): + for post_file in posts_dir.glob("*.json"): + try: + with open(post_file, "r", encoding="utf-8") as f: + post_data = json.load(f) + post_uuid = post_data.get("uuid") + if post_uuid: + self.post_cache[post_uuid] = post_data + except (json.JSONDecodeError, IOError) as e: + logger.debug(f"Error reading post file {post_file}: {e}") + + if comments_dir.exists(): + for comment_file in comments_dir.glob("*.json"): + try: + with open(comment_file, "r", encoding="utf-8") as f: + comment_data = json.load(f) + post_uuid = comment_data.get("post_uuid") + if post_uuid: + self.comment_cache[post_uuid].append(comment_data) + except (json.JSONDecodeError, IOError) as e: + logger.debug(f"Error reading comment file {comment_file}: {e}") + + @staticmethod + def _post_to_dict(post): + return { + "uuid": post.uuid, + "id": post.external_id or post.uuid, + "platform": post.platform, + "source": post.source, + "title": post.title, + "author": post.author, + "url": post.url, + "content": post.content, + "score": post.score, + "timestamp": post.timestamp, + "tags": post.tags or [], + "moderation_uuid": post.moderation_uuid, + } + + @staticmethod + def _comment_to_dict(comment): + return { + "uuid": comment.uuid, + "post_uuid": comment.post_uuid, + "platform": comment.platform, + "parent_comment_uuid": comment.parent_comment_uuid, + "id": comment.comment_id or comment.uuid, + "comment_id": comment.comment_id, + "author": comment.author, + "content": comment.content, + "score": comment.score, + "timestamp": comment.timestamp, + "depth": comment.depth, + "moderation_uuid": comment.moderation_uuid, + } + + def invalidate(self): + """Force the next ``load()`` to refresh content.""" + self.cache_timestamp = 0 + + def quick_stats(self): + """Return {posts_today, total_posts} for the dashboard.""" + cached_posts, _ = self.load() + now = datetime.utcnow() + today_timestamp = (now - timedelta(hours=24)).timestamp() + posts_today = sum( + 1 for post in cached_posts.values() + if post.get("timestamp", 0) >= today_timestamp + ) + return {"posts_today": posts_today, "total_posts": len(cached_posts)} + + @staticmethod + def build_comment_tree(comments): + """Build a hierarchical comment tree from a flat comment list.""" + comment_dict = {c["uuid"]: {**c, "replies": []} for c in comments} + root_comments = [] + for comment in comments: + parent_uuid = comment.get("parent_comment_uuid") + if parent_uuid and parent_uuid in comment_dict: + comment_dict[parent_uuid]["replies"].append( + comment_dict[comment["uuid"]] + ) + else: + root_comments.append(comment_dict[comment["uuid"]]) + + def sort_tree(comments_list): + comments_list.sort(key=lambda x: x.get("timestamp", 0)) + for comment in comments_list: + if comment.get("replies"): + sort_tree(comment["replies"]) + + sort_tree(root_comments) + return root_comments + + def latest_content_mtime(self): + """Latest content update timestamp for client auto-refresh polling.""" + try: + latest_created = Post.query.with_entities(func.max(Post.created_at)).scalar() + if latest_created: + return latest_created.timestamp() + except Exception as e: + logger.debug(f"Could not read latest content timestamp from DB: {e}") + + posts_dir = Path("data/posts") + if not posts_dir.exists(): + return 0 + latest = 0 + for post_file in posts_dir.glob("*.json"): + mtime = post_file.stat().st_mtime + if mtime > latest: + latest = mtime + return latest + + def source_counts(self): + """Count posts per ``platform:source`` for the platforms API.""" + try: + rows = ( + Post.query.with_entities(Post.platform, Post.source, func.count(Post.uuid)) + .group_by(Post.platform, Post.source) + .all() + ) + if rows: + return {f"{platform}:{source}": count for platform, source, count in rows} + except Exception as e: + logger.debug(f"Could not read source counts from DB: {e}") + + counts = {} + posts_dir = Path("data/posts") + if not posts_dir.exists(): + return counts + for post_file in posts_dir.glob("*.json"): + try: + with open(post_file, "r", encoding="utf-8") as f: + post_data = json.load(f) + key = f"{post_data.get('platform', 'unknown')}:{post_data.get('source', '')}" + counts[key] = counts.get(key, 0) + 1 + except (json.JSONDecodeError, IOError): + continue + return counts + + +post_service = PostService() \ No newline at end of file diff --git a/services/settings_service.py b/services/settings_service.py new file mode 100644 index 0000000..99c97da --- /dev/null +++ b/services/settings_service.py @@ -0,0 +1,92 @@ +"""User settings validation and defaults.""" + +import json +import logging +import re + +from config import MAX_COMMUNITY_NAME_LENGTH +from security import is_safe_filterset + +logger = logging.getLogger(__name__) + + +class SettingsService: + """Parse and sanitize user settings JSON shared by page and API routes.""" + + EXPERIENCE_DEFAULTS = { + "infinite_scroll": False, + "auto_refresh": False, + "push_notifications": False, + "dark_patterns_opt_in": False, + "time_filter_enabled": False, + "time_filter_days": 7, + } + + EXPERIENCE_BOOL_FIELDS = { + "infinite_scroll", + "auto_refresh", + "push_notifications", + "dark_patterns_opt_in", + "time_filter_enabled", + } + + @classmethod + def parse(cls, settings_str): + """Return settings JSON as a dict, or an empty dict if invalid.""" + if not settings_str: + return {} + try: + settings = json.loads(settings_str) + except json.JSONDecodeError as e: + logger.warning(f"Invalid user settings JSON: {e}") + return {} + if not isinstance(settings, dict): + logger.warning("User settings must be a JSON object") + return {} + return settings + + @classmethod + def validate(cls, settings_str): + """Validate and sanitize persisted user settings JSON.""" + settings = cls.parse(settings_str) + validated = {} + + filter_set = settings.get("filter_set") + if isinstance(filter_set, str) and is_safe_filterset(filter_set): + validated["filter_set"] = filter_set + + communities = settings.get("communities") + if isinstance(communities, list): + safe_communities = [] + for community in communities: + if ( + isinstance(community, str) + and len(community) <= MAX_COMMUNITY_NAME_LENGTH + and re.match(r"^[a-zA-Z0-9_-]+$", community) + ): + safe_communities.append(community) + validated["communities"] = safe_communities + + experience = settings.get("experience") + if isinstance(experience, dict): + safe_experience = {} + for field in cls.EXPERIENCE_BOOL_FIELDS: + if field in experience and isinstance(experience[field], bool): + safe_experience[field] = experience[field] + + time_filter_days = experience.get("time_filter_days") + if isinstance(time_filter_days, int) and time_filter_days > 0: + safe_experience["time_filter_days"] = time_filter_days + + validated["experience"] = safe_experience + + return validated + + @classmethod + def experience_settings(cls, settings_str): + """Return experience settings with defaults filled in.""" + settings = cls.parse(settings_str) + experience = settings.get("experience", {}) + if not isinstance(experience, dict): + experience = {} + return {**cls.EXPERIENCE_DEFAULTS, **experience} diff --git a/start_server.py b/start_server.py index 38913dc..ef8bc59 100755 --- a/start_server.py +++ b/start_server.py @@ -127,13 +127,14 @@ def start_flask(): # Import and run Flask app try: - from app import app + from app import create_app print_color("✓ Flask app imported successfully", 'green') print_color("✓ Database initialized with SQLAlchemy", 'green') print_color("✓ User authentication ready", 'green') print() # Run Flask + app = create_app() app.run(host='0.0.0.0', port=FLASK_PORT, debug=True, use_reloader=False) except Exception as e: diff --git a/templates/admin.html b/templates/admin.html index d434009..c95de49 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -171,16 +171,6 @@
- -
-

Content Actions

-
- -

- This will regenerate all HTML files with current templates and filters. -

-
-
diff --git a/templates/bookmarks.html b/templates/bookmarks.html index 3fac042..12942ae 100644 --- a/templates/bookmarks.html +++ b/templates/bookmarks.html @@ -136,7 +136,7 @@ async function loadBookmarks(page = 1) { try { document.getElementById('loading').style.display = 'block'; - const response = await fetch(`/api/bookmarks?page=${page}&per_page=20`); + const response = await fetch(`/api/v1/bookmarks?page=${page}&per_page=20`); const data = await response.json(); if (!response.ok) { @@ -226,7 +226,7 @@ async function removeBookmark(postId, button) { button.disabled = true; button.textContent = 'Removing...'; - const response = await fetch('/api/bookmark', { + const response = await fetch('/api/v1/bookmark', { method: 'POST', headers: { 'Content-Type': 'application/json', diff --git a/templates/dashboard.html b/templates/dashboard.html index 77d13bc..9ca1f14 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -708,7 +708,7 @@ document.addEventListener('DOMContentLoaded', function() { // Load platform configuration and communities async function loadPlatformConfig() { try { - const response = await fetch('/api/platforms'); + const response = await fetch('/api/v1/platforms'); if (!response.ok) { throw new Error(`HTTP error! status: ${response.status}`); } @@ -736,7 +736,7 @@ async function loadPlatformConfig() { // Load available filters async function loadFilters() { try { - const response = await fetch('/api/filters'); + const response = await fetch('/api/v1/filters'); const data = await response.json(); filtersData = data.filters || []; @@ -832,7 +832,7 @@ async function loadPosts(page = 1, community = '', platform = '', append = false if (filter || currentFilter) params.append('filter', filter || currentFilter); if (currentSearchQuery) params.append('q', currentSearchQuery); - const response = await fetch(`/api/posts?${params}`); + const response = await fetch(`/api/v1/posts?${params}`); const data = await response.json(); const newPosts = data.posts || []; paginationData = data.pagination || {}; @@ -1195,7 +1195,7 @@ function setupAutoRefresh() { if (currentPage === 1 && !currentCommunity && !currentPlatform) { try { // Check if new content is available by checking timestamp - const response = await fetch('/api/content-timestamp'); + const response = await fetch('/api/v1/content-timestamp'); const data = await response.json(); const lastContentUpdate = data.timestamp; diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..15b5aca --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,112 @@ +"""Shared pytest fixtures for BalanceBoard. + +The app fixture runs against an in-memory SQLite database (no Postgres +required) by monkeypatching ``database.init_db``. Filter-engine and +polling-service singletons are stubbed so request handlers do not start +background threads or hit the real filter pipeline during API contract +tests. +""" + +import pytest +from sqlalchemy.pool import StaticPool + + +def _sqlite_init_db(app): + """Replace the Postgres init_db with an in-memory SQLite setup.""" + from database import db + + app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:" + # A single in-memory DB shared across the test's connections. + app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"poolclass": StaticPool} + app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False + db.init_app(app) + with app.app_context(): + db.create_all() + + +@pytest.fixture +def app(monkeypatch): + import database + from app import create_app + + monkeypatch.setattr(database, "init_db", _sqlite_init_db) + application = create_app() + application.config.update(TESTING=True) + return application + + +@pytest.fixture +def client(app): + return app.test_client() + + +class _StubPolling: + """No-op polling service so ``before_request`` does not start threads.""" + + def init_app(self, app): + pass + + def start(self): + pass + + +class _StubFilterEngine: + """Passthrough filter engine for API contract tests.""" + + class _Config: + def get_filterset(self, name): + return None + + config = _Config() + + def apply_filterset(self, posts, filterset_name="no_filter", use_cache=True): + for p in posts: + p.setdefault("_filter_score", 0.5) + p.setdefault("_filter_categories", []) + p.setdefault("_filter_tags", []) + return posts + + def filter_comments(self, comments, filterset_name="no_filter"): + return comments + + def get_available_filtersets(self): + return ["no_filter"] + + +@pytest.fixture +def stub_services(app, monkeypatch): + """Patch the app's lazy service accessors so requests stay hermetic.""" + import app as app_module + + monkeypatch.setattr(app_module, "get_filter_engine", lambda: _StubFilterEngine()) + monkeypatch.setattr(app_module, "get_polling_service", lambda: _StubPolling()) + return app + + +class StubPostService: + """In-memory post/comment store for /api/v1 contract tests.""" + + def __init__(self, posts=None, comments=None): + self._posts = posts or {} + self._comments = comments or {} + + def load(self): + return self._posts, self._comments + + @staticmethod + def build_comment_tree(comments): + comment_dict = {c["uuid"]: {**c, "replies": []} for c in comments} + roots = [] + for c in comments: + parent = c.get("parent_comment_uuid") + if parent and parent in comment_dict: + comment_dict[parent]["replies"].append(comment_dict[c["uuid"]]) + else: + roots.append(comment_dict[c["uuid"]]) + return roots + + def source_counts(self): + return {} + + def latest_content_mtime(self): + return 0 \ No newline at end of file diff --git a/tests/test_api_contracts.py b/tests/test_api_contracts.py new file mode 100644 index 0000000..3542527 --- /dev/null +++ b/tests/test_api_contracts.py @@ -0,0 +1,102 @@ +"""/api/v1 contract tests with monkeypatched post_service + filter engine. + +No live Postgres and no real filter pipeline: ``post_service`` is replaced with +an in-memory stub and ``get_filter_engine`` with a passthrough stub. These +assert the JSON shape the templates/SPA consume, so refactor regressions are +caught without a runtime stack. +""" + +import pytest + +from conftest import StubPostService, _StubFilterEngine + + +SAMPLE_POST = { + "uuid": "post-1", + "title": "Sample post", + "author": "alice", + "platform": "hackernews", + "source": "programming", + "score": 42, + "timestamp": 1700000000, + "url": "https://example.com/1", + "content": "Hello world", + "tags": ["tech"], +} + +SAMPLE_COMMENTS = [ + {"uuid": "c1", "post_uuid": "post-1", "content": "top", "score": 3, + "depth": 0, "parent_comment_uuid": None}, + {"uuid": "c2", "post_uuid": "post-1", "content": "reply", "score": 1, + "depth": 1, "parent_comment_uuid": "c1"}, +] + + +@pytest.fixture +def api_client(app, monkeypatch): + """App + client with stubbed post_service and filter engine.""" + import app as app_module + import blueprints.api as api_module + + monkeypatch.setattr(app_module, "get_filter_engine", lambda: _StubFilterEngine()) + monkeypatch.setattr(app_module, "get_polling_service", lambda: _StubPolling()) + monkeypatch.setattr(api_module, "get_filter_engine", lambda: _StubFilterEngine()) + monkeypatch.setattr( + api_module, "post_service", + StubPostService(posts={"post-1": SAMPLE_POST}, comments={"post-1": SAMPLE_COMMENTS}), + ) + return app.test_client() + + +def test_posts_endpoint_returns_paginated_shape(api_client): + resp = api_client.get("/api/v1/posts") + assert resp.status_code == 200 + data = resp.get_json() + assert set(data.keys()) >= {"posts", "pagination"} + assert set(data["pagination"].keys()) >= { + "current_page", "total_pages", "total_posts", "per_page", "has_next", "has_prev" + } + assert data["pagination"]["total_posts"] == 1 + post = data["posts"][0] + assert post["id"] == "post-1" + assert post["title"] == "Sample post" + assert post["platform"] == "hackernews" + assert post["url"] == "/post/post-1" + assert "filter_score" in post + + +def test_post_detail_returns_post_and_comments(api_client): + resp = api_client.get("/api/v1/posts/post-1") + assert resp.status_code == 200 + data = resp.get_json() + assert data["post"]["uuid"] == "post-1" + assert isinstance(data["comments"], list) + # The tree has one root with one nested reply. + assert data["comments"][0]["uuid"] == "c1" + assert data["comments"][0]["replies"][0]["uuid"] == "c2" + + +def test_post_detail_404_for_unknown(api_client): + resp = api_client.get("/api/v1/posts/does-not-exist") + assert resp.status_code == 404 + + +def test_comments_endpoint_returns_tree(api_client): + resp = api_client.get("/api/v1/comments/post-1") + assert resp.status_code == 200 + data = resp.get_json() + assert data["comments"][0]["uuid"] == "c1" + assert data["comments"][0]["replies"][0]["uuid"] == "c2" + + +def test_filters_endpoint_lists_filtersets(api_client): + resp = api_client.get("/api/v1/filters") + assert resp.status_code == 200 + data = resp.get_json() + assert "filters" in data + # Stub engine advertises no_filter; the list may be empty or contain it. + assert isinstance(data["filters"], list) + + +# Imported via the api_client fixture's monkeypatch; keep the name available. +from conftest import _StubPolling # noqa: E402, F401 \ No newline at end of file diff --git a/tests/test_app_factory.py b/tests/test_app_factory.py new file mode 100644 index 0000000..48c226a --- /dev/null +++ b/tests/test_app_factory.py @@ -0,0 +1,58 @@ +"""App-factory tests: route registration and endpoint names are intact. + +These guard the Phase 1 refactor (module-level ``app`` → ``create_app()`` with +routes split under ``routes/`` and ``blueprints/``). They assert that the +endpoints referenced by templates via ``url_for(...)`` still resolve in the +Flask url_map. Runs against in-memory SQLite (no Postgres). +""" + +import pytest + +# Endpoints that templates reference via url_for(...) and that must survive +# the factory refactor. Sourced from the Jinja templates. +TEMPLATE_ENDPOINTS = [ + "login", + "signup", + "admin_setup", + "logout", + "serve_logo", + "serve_theme", + "static", + # API blueprint endpoints (mounted at /api/v1) + "api.posts", + "api.post_detail", + "api.comments", + "api.filters", + "api.bookmarks", + "api.platforms", +] + + +def test_create_app_returns_flask_app(app): + from flask import Flask + + assert isinstance(app, Flask) + + +def test_api_blueprint_mounted_under_v1(app): + rules = [r.rule for r in app.url_map.iter_rules()] + assert any(r.startswith("/api/v1/posts") for r in rules), rules + assert any(r.startswith("/api/v1/comments") for r in rules), rules + + +@pytest.mark.parametrize("endpoint", TEMPLATE_ENDPOINTS) +def test_template_endpoints_exist(app, endpoint): + # Flask stores endpoints as "." for blueprint views and + # bare names for app-level views. ``url_map`` has both. + all_endpoints = {r.endpoint for r in app.url_map.iter_rules()} + assert endpoint in all_endpoints, ( + f"endpoint '{endpoint}' missing from url_map; have: {sorted(all_endpoints)[:20]}..." + ) + + +def test_no_module_level_app_object(): + """Phase 1 removed the module-level ``app``; importing app.py must not + start a server or expose a global Flask app.""" + import app as app_module + + assert not hasattr(app_module, "app"), "app.py must not keep a module-level `app`" \ No newline at end of file diff --git a/tests/test_filter_pipeline.py b/tests/test_filter_pipeline.py new file mode 100644 index 0000000..564d7dd --- /dev/null +++ b/tests/test_filter_pipeline.py @@ -0,0 +1,103 @@ +"""Offline filter-pipeline tests. + +These do NOT require Flask or Postgres — only the filter_pipeline package (which +depends on stdlib + ``requests``). They exercise the registry-driven engine, +the offline plugin path, AI-disabled fail-open behavior, and comment filtering. +""" + +import pytest + +from filter_pipeline.engine import FilterEngine +from filter_pipeline.models import ProcessingStatus +from filter_pipeline.registry import ( + get_registered_plugins, + get_registered_stages, +) + + +@pytest.fixture +def engine(): + # Fresh engine (not the singleton) so test isolation holds. + return FilterEngine("filter_config.json", "filtersets.json") + + +def test_registry_discovers_builtin_stages_and_plugins(engine): + engine._init_stages() + stages = get_registered_stages() + for name in ["categorizer", "moderator", "filter", "ranker", + "plugins", "comment_filter"]: + assert name in stages, f"stage '{name}' not registered" + plugins = get_registered_plugins() + for name in ["keyword", "quality"]: + assert name in plugins, f"plugin '{name}' not registered" + + +def test_offline_filterset_runs_without_ai(engine): + # quality_filter uses pipeline_stages=['plugins','ranker'] only. + posts = [ + {"uuid": "p1", "title": "A fine Python programming title", + "content": "c" * 200, "score": 10, "replies": 2, + "platform": "reddit", "source": "python", "timestamp": 1700000000}, + {"uuid": "p2", "title": "bad", "content": "x", "score": 0, "replies": 0, + "platform": "reddit", "source": "python", "timestamp": 1700000000}, + ] + results = engine.process_batch(posts, "quality_filter") + assert results[0].passed is True + # Quality plugin rejects the 3-char title. + assert results[1].passed is False + assert any("QualityFilter" in t for t in results[1].tags) + + +def test_ai_filterset_does_not_blank_feed_when_ai_disabled(engine): + posts = [{"uuid": "p1", "title": "Hello world this is a fine title", + "content": "c" * 200, "score": 10, "replies": 2, + "platform": "hackernews", "source": "programming", + "timestamp": 1700000000}] + out = engine.apply_filterset(posts, "safe_content", use_cache=False) + assert len(out) == 1, "AI-disabled filterset must not blank the feed" + results = engine.process_batch(posts, "safe_content") + assert results[0].status == ProcessingStatus.FAILED + + +def test_no_filter_passes_everything(engine): + posts = [{"uuid": "p1", "title": "anything", "content": "", "score": 0, + "timestamp": 0}] + out = engine.apply_filterset(posts, "no_filter", use_cache=False) + assert len(out) == 1 + + +def test_comment_filter_individual_mode(engine): + comments = [ + {"uuid": "c1", "content": "this is long enough", "score": 5, + "depth": 0, "parent_comment_uuid": None}, + {"uuid": "c2", "content": "hi", "score": 1, "depth": 1, + "parent_comment_uuid": "c1"}, + ] + kept = engine.filter_comments(comments, "quality_filter") + assert [c["uuid"] for c in kept] == ["c1"] + + +def test_comment_filter_no_filter_passes_all(engine): + comments = [{"uuid": "c1", "content": "hi", "score": 1, "depth": 0, + "parent_comment_uuid": None}] + assert len(engine.filter_comments(comments, "no_filter")) == 1 + + +def test_comment_filter_unknown_filterset_fails_open(engine): + comments = [{"uuid": "c1", "content": "hi", "score": 1, "depth": 0, + "parent_comment_uuid": None}] + assert len(engine.filter_comments(comments, "no_such_filterset")) == 1 + + +def test_comment_tree_pruning_drops_branch_without_moderation(engine): + # safe_content comment rules require moderation.flags.is_safe == True. + # With no moderation data attached the field is None -> rule fails closed, + # so tree pruning removes the parent and its child. + comments = [ + {"uuid": "r", "content": "root", "score": 5, "depth": 0, + "parent_comment_uuid": None}, + {"uuid": "c", "content": "child", "score": 1, "depth": 1, + "parent_comment_uuid": "r"}, + ] + kept = engine.filter_comments(comments, "safe_content") + assert kept == [] \ No newline at end of file diff --git a/tests/test_plugin_contract.py b/tests/test_plugin_contract.py new file mode 100644 index 0000000..df9ec54 --- /dev/null +++ b/tests/test_plugin_contract.py @@ -0,0 +1,90 @@ +"""Plugin/stage contract test (Phase 6). + +A drop-in stage and plugin, defined ONLY in this test module via the public +``@register_stage`` / ``@register_plugin`` decorators, are picked up by the +engine with zero edits to core files. This is the pluggability guarantee: a +new filter behavior is a new module + a config entry, never an edit to +``engine.py``. +""" + +from filter_pipeline.engine import FilterEngine +from filter_pipeline.models import FilterResult +from filter_pipeline.plugins.base import BaseFilterPlugin +from filter_pipeline.registry import ( + get_plugin_class, + get_stage_class, + register_plugin, + register_stage, +) +from filter_pipeline.stages.base_stage import BaseStage + + +@register_stage("sentinel_dropin_stage") +class SentinelStage(BaseStage): + """Drop-in stage that tags any result it sees.""" + + def get_name(self): + return "Sentinel" + + def process(self, post, result): + result.tags.append("sentinel_ran") + return result + + +@register_plugin("sentinel_dropin_plugin") +class SentinelPlugin(BaseFilterPlugin): + """Drop-in plugin: never rejects, returns a fixed score.""" + + def get_name(self): + return "SentinelPlugin" + + def should_filter(self, post, context=None): + return False + + def score(self, post, context=None): + return 0.9 + + +def test_dropin_stage_is_registered(): + assert get_stage_class("sentinel_dropin_stage") is SentinelStage + + +def test_dropin_plugin_is_registered(): + assert get_plugin_class("sentinel_dropin_plugin") is SentinelPlugin + + +def test_engine_instantiates_dropin_stage(): + eng = FilterEngine("filter_config.json", "filtersets.json") + eng._init_stages() + assert "sentinel_dropin_stage" in eng._stages + stage = eng._stages["sentinel_dropin_stage"] + # Running the stage through the contract it claims to implement works. + result = FilterResult(post_uuid="x", passed=True, score=0.5) + out = stage.process({"uuid": "x"}, result) + assert "sentinel_ran" in out.tags + + +def test_dropin_stage_can_be_selected_in_a_filterset(tmp_path): + """A filterset that lists the drop-in stage actually runs it. + + Builds a throwaway config + filterset on disk so no core file is edited. + """ + import json + + cfg = tmp_path / "cfg.json" + cfg.write_text(json.dumps({ + "ai": {"enabled": False}, + "cache": {"enabled": False}, + "pipeline": {"default_stages": ["sentinel_dropin_stage"], "enable_parallel": False}, + "plugins": {"enabled": [], "configs": {}}, + })) + fs = tmp_path / "fs.json" + fs.write_text(json.dumps({"custom": {"post_rules": {}, "comment_rules": {}}})) + + eng = FilterEngine(str(cfg), str(fs)) + eng._init_stages() + results = eng.process_batch( + [{"uuid": "p1", "title": "t", "content": "", "score": 0, "timestamp": 0}], + "custom", + ) + assert "sentinel_ran" in results[0].tags \ No newline at end of file diff --git a/themes/modern-card-ui/index.html b/themes/modern-card-ui/index.html index 74d8789..ffe6749 100644 --- a/themes/modern-card-ui/index.html +++ b/themes/modern-card-ui/index.html @@ -363,7 +363,7 @@ const originalText = button.querySelector('.bookmark-text').textContent; button.querySelector('.bookmark-text').textContent = 'Saving...'; - const response = await fetch('/api/bookmark', { + const response = await fetch('/api/v1/bookmark', { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -412,7 +412,7 @@ const postId = button.getAttribute('data-post-id'); try { - const response = await fetch(`/api/bookmark-status/${postId}`); + const response = await fetch(`/api/v1/bookmark-status/${postId}`); const data = await response.json(); if (response.ok && data.bookmarked) { diff --git a/themes/template_prompt.txt b/themes/template_prompt.txt deleted file mode 100644 index 12a097d..0000000 --- a/themes/template_prompt.txt +++ /dev/null @@ -1,120 +0,0 @@ -# Template Creation Prompt for AI - -This document describes the data structures, helper functions, and conventions an AI needs to create or modify HTML templates for this social media archive system. - -## Data Structures Available - -### Post Data (when rendering posts) -- **Available in all post templates (card, list, detail):** - - platform: string (e.g., "reddit", "hackernews") - - id: string (unique post identifier) - - title: string - - author: string - - timestamp: integer (unix timestamp) - - score: integer (up/down vote score) - - replies: integer (number of comments) - - url: string (original post URL) - - content: string (optional post body text) - - source: string (optional subreddit/community) - - tags: array of strings (optional tags/flair) - - meta: object (optional platform-specific metadata) - - comments: array (optional nested comment tree - only in detail templates) - - post_url: string (generated: "{uuid}.html" - for local linking to detail pages) - -### Comment Data (when rendering comments) -- **Available in comment templates:** - - uuid: string (unique comment identifier) - - id: string (platform-specific identifier) - - author: string (comment author username) - - content: string (comment text) - - timestamp: integer (unix timestamp) - - score: integer (comment score) - - platform: string - - depth: integer (nesting level) - - children: array (nested replies) - - children_section: string (pre-rendered HTML of nested children) - -## Template Engine: Jinja2 - -Templates use Jinja2 syntax (`{{ }}` for variables, `{% %}` for control flow). - -### Important Filters: -- `|safe`: Mark content as safe HTML (for already-escaped content) -- Example: `{{ renderMarkdown(content)|safe }}` - -### Available Control Structures: -- `{% if variable %}...{% endif %}` -- `{% for item in array %}...{% endfor %}` -- `{% set variable = value %}` (create local variables) - -## Helper Functions Available - -Call these in templates using `{{ function(arg) }}`: - -### Time/Date Formatting: -- `formatTime(timestamp)` -> "HH:MM" -- `formatTimeAgo(timestamp)` -> "2 hours ago" -- `formatDateTime(timestamp)` -> "January 15, 2024 at 14:30" - -### Text Processing: -- `truncate(text, max_length)` -> truncated string with "..." -- `escapeHtml(text)` -> HTML-escaped version - -### Content Rendering: -- `renderMarkdown(text)` -> Basic HTML from markdown (returns already-escaped HTML) - -## Template Types - -### Card Template (for index/listing pages) -- Used for summary view of posts -- Links should use `post_url` to point to local detail pages -- Keep concise - truncated content, basic info - -### List Template (compact listing) -- Even more compact than cards -- Vote scores, basic metadata, title link - -### Detail Template (full post view) -- Full content, meta information -- Source link uses `url` (external) -- Must include `{{comments_section|safe}}` for rendered comments - -### Comment Template (nested comments) -- Recursive rendering with depth styling -- Children rendered as flattened HTML in `children_section` - -## Convenience Data Added by System - -In `generate_html.py`, `post_url` is added to each post before rendering: `{post['uuid']}.html` - -This allows templates to link to local detail pages instead of external Reddit. - -## CSS Classes Convention - -Templates use semantic CSS classes: -- Post cards: `.post-card`, `.post-header`, `.post-meta`, etc. -- Comments: `.comment`, `.comment-header`, `.comment-body`, etc. -- Platform: `.platform-{platform}` for platform-specific styling - -## Examples - -### Conditional Rendering: -``` -{% if content %} -

{{ renderMarkdown(content)|safe }}

-{% endif %} -``` - -### Looping Tags: -``` -{% for tag in tags if tag %} - {{ tag }} -{% endfor %} -``` - -### Styling by Depth (comments): -``` -
-``` - -When creating new templates, follow these patterns and use the available data and helper functions appropriately.