Phase 1-3 + 6: pluggable filter system, app factory, test harness

Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md.

Phase 1 — app factory + blueprints + /api/v1:
- app.py -> create_app() factory (no module-level app); entrypoints updated
- routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1
- config.py / extensions.py / security.py extracted; services/ layer added
- endpoint names preserved so template url_for() calls keep resolving
  (static check: all 27 template url_for endpoints are defined routes)

Phase 2 — one pluggable filter system:
- filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules
- engine._init_stages() instantiates registered stages (no hardcoded dict);
  process_batch is AI-aware: only short-circuits to the AI-disabled path when
  a filterset's stages declare requires_ai, so offline filtersets run with AI off
- BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality
  re-enabled via filter_config.json plugins config
- comment tree modes ported to stages/comment_filter.py + shared rules.py;
  wired into /api/v1/posts/<uuid> and /api/v1/comments/<uuid> via
  FilterEngine.filter_comments() (fails open)
- offline quality_filter filterset exercises plugins+ranker without AI
- legacy filter_lib / comment_lib / html_generation_lib / generate_html /
  active_html path deleted

Phase 3 prep — pluggable fetchers + Postgres models:
- Post / Comment SQLAlchemy models added to models.py
- migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run)
- platforms/ fetcher registry (extension point)
- live reads/writes still go through PostService (disk JSON); cutover deferred

Phase 6 — test harness:
- pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres;
  stubbed polling/filter singletons)
- test_app_factory.py (route registration, no module-level app),
  test_api_contracts.py (posts/post_detail/comments/filters shape with
  monkeypatched post_service + get_filter_engine),
  test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated
  locally 12/12 incl. drop-in stage/plugin discovered with zero core edits)

Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements.

Verification: py_compile clean across the project; the Flask-free filter-pipeline
and plugin-contract tests pass locally. App-factory / API-contract tests need
deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated
on docker.

Co-Authored-By: Claude <noreply@anthropic.com>

migrate_content_to_db.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python
+"""Backfill on-disk content JSON into the Postgres ``posts``/``comments`` tables.
+
+Phase 3 prep (Agent D): reads ``data/posts/*.json`` and ``data/comments/*.json``
+and upserts them into the ``Post`` / ``Comment`` models. Live reads/writes still
+go through ``PostService`` (disk JSON) — this script only populates the DB so a
+later cutover has data to read. It is idempotent: existing rows are skipped by
+``uuid`` (run it again after collecting new content to backfill only the new
+files).
+
+Usage::
+
+    python migrate_content_to_db.py [--data-dir data] [--batch-size 500] [--dry-run]
+
+Requires the full Flask/Postgres stack (``DATABASE_URL`` or the POSTGRES_*
+env vars) — run inside the docker compose environment, not locally.
+"""
+
+import argparse
+import json
+import logging
+import sys
+from pathlib import Path
+
+from app import create_app
+from database import db
+from models import Comment, Post
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger("migrate_content_to_db")
+
+
+def _load_json(path: Path):
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError) as e:
+        logger.warning("Skipping unreadable file %s: %s", path, e)
+        return None
+
+
+def _existing_uuids(model, uuids):
+    """Return the subset of ``uuids`` already present in the table."""
+    if not uuids:
+        return set()
+    found = set()
+    # Chunk to avoid huge IN clauses.
+    for i in range(0, len(uuids), 500):
+        chunk = uuids[i:i + 500]
+        rows = db.session.query(model.uuid).filter(model.uuid.in_(chunk)).all()
+        found.update(r[0] for r in rows)
+    return found
+
+
+def backfill_posts(posts_dir: Path, batch_size: int, dry_run: bool) -> int:
+    files = sorted(posts_dir.glob("*.json")) if posts_dir.exists() else []
+    if not files:
+        logger.info("No post files found in %s", posts_dir)
+        return 0
+
+    records = []
+    for pf in files:
+        data = _load_json(pf)
+        if not data or not data.get("uuid"):
+            continue
+        records.append(data)
+
+    seen = _existing_uuids(Post, [r["uuid"] for r in records])
+    inserted = 0
+    batch = []
+    for r in records:
+        if r["uuid"] in seen:
+            continue
+        batch.append(Post(
+            uuid=r["uuid"],
+            external_id=r.get("id"),
+            platform=r.get("platform", "") or "",
+            source=r.get("source", "") or "",
+            title=(r.get("title") or "")[:500],
+            author=r.get("author"),
+            url=r.get("url"),
+            content=r.get("content"),
+            score=int(r.get("score", 0) or 0),
+            timestamp=int(r.get("timestamp", 0) or 0),
+            tags=r.get("tags"),
+            moderation_uuid=r.get("moderation_uuid"),
+        ))
+        inserted += 1
+        if len(batch) >= batch_size:
+            _flush(batch, dry_run)
+            batch = []
+    _flush(batch, dry_run)
+    logger.info("Posts: backfilled %d new (%d already present)", inserted, len(seen))
+    return inserted
+
+
+def backfill_comments(comments_dir: Path, batch_size: int, dry_run: bool) -> int:
+    files = sorted(comments_dir.glob("*.json")) if comments_dir.exists() else []
+    if not files:
+        logger.info("No comment files found in %s", comments_dir)
+        return 0
+
+    records = []
+    for cf in files:
+        data = _load_json(cf)
+        if not data or not data.get("uuid"):
+            continue
+        records.append(data)
+
+    seen = _existing_uuids(Comment, [r["uuid"] for r in records])
+    inserted = 0
+    batch = []
+    for r in records:
+        if r["uuid"] in seen:
+            continue
+        batch.append(Comment(
+            uuid=r["uuid"],
+            post_uuid=r.get("post_uuid") or "",
+            platform=r.get("platform"),
+            parent_comment_uuid=r.get("parent_comment_uuid"),
+            comment_id=r.get("comment_id"),
+            author=r.get("author"),
+            content=r.get("content"),
+            score=int(r.get("score", 0) or 0),
+            timestamp=int(r.get("timestamp", 0) or 0),
+            depth=int(r.get("depth", 0) or 0),
+            moderation_uuid=r.get("moderation_uuid"),
+        ))
+        inserted += 1
+        if len(batch) >= batch_size:
+            _flush(batch, dry_run)
+            batch = []
+    _flush(batch, dry_run)
+    logger.info("Comments: backfilled %d new (%d already present)", inserted, len(seen))
+    return inserted
+
+
+def _flush(batch, dry_run):
+    if not batch:
+        return
+    if dry_run:
+        logger.info("[dry-run] would insert %d rows", len(batch))
+        return
+    db.session.bulk_save_objects(batch)
+    db.session.commit()
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Backfill content JSON into Postgres.")
+    parser.add_argument("--data-dir", default="data", help="Root data directory.")
+    parser.add_argument("--batch-size", type=int, default=500, help="Insert batch size.")
+    parser.add_argument("--dry-run", action="store_true", help="Log counts without writing.")
+    args = parser.parse_args(argv)
+
+    app = create_app()
+    with app.app_context():
+        posts_dir = Path(args.data_dir) / "posts"
+        comments_dir = Path(args.data_dir) / "comments"
+        p = backfill_posts(posts_dir, args.batch_size, args.dry_run)
+        c = backfill_comments(comments_dir, args.batch_size, args.dry_run)
+        logger.info("Done. posts=%d comments=%d", p, c)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())