Phase 1-3 + 6: pluggable filter system, app factory, test harness

Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md.

Phase 1 — app factory + blueprints + /api/v1:
- app.py -> create_app() factory (no module-level app); entrypoints updated
- routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1
- config.py / extensions.py / security.py extracted; services/ layer added
- endpoint names preserved so template url_for() calls keep resolving
  (static check: all 27 template url_for endpoints are defined routes)

Phase 2 — one pluggable filter system:
- filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules
- engine._init_stages() instantiates registered stages (no hardcoded dict);
  process_batch is AI-aware: only short-circuits to the AI-disabled path when
  a filterset's stages declare requires_ai, so offline filtersets run with AI off
- BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality
  re-enabled via filter_config.json plugins config
- comment tree modes ported to stages/comment_filter.py + shared rules.py;
  wired into /api/v1/posts/<uuid> and /api/v1/comments/<uuid> via
  FilterEngine.filter_comments() (fails open)
- offline quality_filter filterset exercises plugins+ranker without AI
- legacy filter_lib / comment_lib / html_generation_lib / generate_html /
  active_html path deleted

Phase 3 prep — pluggable fetchers + Postgres models:
- Post / Comment SQLAlchemy models added to models.py
- migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run)
- platforms/ fetcher registry (extension point)
- live reads/writes still go through PostService (disk JSON); cutover deferred

Phase 6 — test harness:
- pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres;
  stubbed polling/filter singletons)
- test_app_factory.py (route registration, no module-level app),
  test_api_contracts.py (posts/post_detail/comments/filters shape with
  monkeypatched post_service + get_filter_engine),
  test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated
  locally 12/12 incl. drop-in stage/plugin discovered with zero core edits)

Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements.

Verification: py_compile clean across the project; the Flask-free filter-pipeline
and plugin-contract tests pass locally. App-factory / API-contract tests need
deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated
on docker.

Co-Authored-By: Claude <noreply@anthropic.com>

tests/conftest.py

@@ -0,0 +1,112 @@
+"""Shared pytest fixtures for BalanceBoard.
+
+The app fixture runs against an in-memory SQLite database (no Postgres
+required) by monkeypatching ``database.init_db``. Filter-engine and
+polling-service singletons are stubbed so request handlers do not start
+background threads or hit the real filter pipeline during API contract
+tests.
+"""
+
+import pytest
+from sqlalchemy.pool import StaticPool
+
+
+def _sqlite_init_db(app):
+    """Replace the Postgres init_db with an in-memory SQLite setup."""
+    from database import db
+
+    app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
+    # A single in-memory DB shared across the test's connections.
+    app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"poolclass": StaticPool}
+    app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
+    db.init_app(app)
+    with app.app_context():
+        db.create_all()
+
+
+@pytest.fixture
+def app(monkeypatch):
+    import database
+    from app import create_app
+
+    monkeypatch.setattr(database, "init_db", _sqlite_init_db)
+    application = create_app()
+    application.config.update(TESTING=True)
+    return application
+
+
+@pytest.fixture
+def client(app):
+    return app.test_client()
+
+
+class _StubPolling:
+    """No-op polling service so ``before_request`` does not start threads."""
+
+    def init_app(self, app):
+        pass
+
+    def start(self):
+        pass
+
+
+class _StubFilterEngine:
+    """Passthrough filter engine for API contract tests."""
+
+    class _Config:
+        def get_filterset(self, name):
+            return None
+
+    config = _Config()
+
+    def apply_filterset(self, posts, filterset_name="no_filter", use_cache=True):
+        for p in posts:
+            p.setdefault("_filter_score", 0.5)
+            p.setdefault("_filter_categories", [])
+            p.setdefault("_filter_tags", [])
+        return posts
+
+    def filter_comments(self, comments, filterset_name="no_filter"):
+        return comments
+
+    def get_available_filtersets(self):
+        return ["no_filter"]
+
+
+@pytest.fixture
+def stub_services(app, monkeypatch):
+    """Patch the app's lazy service accessors so requests stay hermetic."""
+    import app as app_module
+
+    monkeypatch.setattr(app_module, "get_filter_engine", lambda: _StubFilterEngine())
+    monkeypatch.setattr(app_module, "get_polling_service", lambda: _StubPolling())
+    return app
+
+
+class StubPostService:
+    """In-memory post/comment store for /api/v1 contract tests."""
+
+    def __init__(self, posts=None, comments=None):
+        self._posts = posts or {}
+        self._comments = comments or {}
+
+    def load(self):
+        return self._posts, self._comments
+
+    @staticmethod
+    def build_comment_tree(comments):
+        comment_dict = {c["uuid"]: {**c, "replies": []} for c in comments}
+        roots = []
+        for c in comments:
+            parent = c.get("parent_comment_uuid")
+            if parent and parent in comment_dict:
+                comment_dict[parent]["replies"].append(comment_dict[c["uuid"]])
+            else:
+                roots.append(comment_dict[c["uuid"]])
+        return roots
+
+    def source_counts(self):
+        return {}
+
+    def latest_content_mtime(self):
+        return 0

tests/test_api_contracts.py

@@ -0,0 +1,102 @@
+"""/api/v1 contract tests with monkeypatched post_service + filter engine.
+
+No live Postgres and no real filter pipeline: ``post_service`` is replaced with
+an in-memory stub and ``get_filter_engine`` with a passthrough stub. These
+assert the JSON shape the templates/SPA consume, so refactor regressions are
+caught without a runtime stack.
+"""
+
+import pytest
+
+from conftest import StubPostService, _StubFilterEngine
+
+
+SAMPLE_POST = {
+    "uuid": "post-1",
+    "title": "Sample post",
+    "author": "alice",
+    "platform": "hackernews",
+    "source": "programming",
+    "score": 42,
+    "timestamp": 1700000000,
+    "url": "https://example.com/1",
+    "content": "Hello world",
+    "tags": ["tech"],
+}
+
+SAMPLE_COMMENTS = [
+    {"uuid": "c1", "post_uuid": "post-1", "content": "top", "score": 3,
+     "depth": 0, "parent_comment_uuid": None},
+    {"uuid": "c2", "post_uuid": "post-1", "content": "reply", "score": 1,
+     "depth": 1, "parent_comment_uuid": "c1"},
+]
+
+
+@pytest.fixture
+def api_client(app, monkeypatch):
+    """App + client with stubbed post_service and filter engine."""
+    import app as app_module
+    import blueprints.api as api_module
+
+    monkeypatch.setattr(app_module, "get_filter_engine", lambda: _StubFilterEngine())
+    monkeypatch.setattr(app_module, "get_polling_service", lambda: _StubPolling())
+    monkeypatch.setattr(api_module, "get_filter_engine", lambda: _StubFilterEngine())
+    monkeypatch.setattr(
+        api_module, "post_service",
+        StubPostService(posts={"post-1": SAMPLE_POST}, comments={"post-1": SAMPLE_COMMENTS}),
+    )
+    return app.test_client()
+
+
+def test_posts_endpoint_returns_paginated_shape(api_client):
+    resp = api_client.get("/api/v1/posts")
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert set(data.keys()) >= {"posts", "pagination"}
+    assert set(data["pagination"].keys()) >= {
+        "current_page", "total_pages", "total_posts", "per_page", "has_next", "has_prev"
+    }
+    assert data["pagination"]["total_posts"] == 1
+    post = data["posts"][0]
+    assert post["id"] == "post-1"
+    assert post["title"] == "Sample post"
+    assert post["platform"] == "hackernews"
+    assert post["url"] == "/post/post-1"
+    assert "filter_score" in post
+
+
+def test_post_detail_returns_post_and_comments(api_client):
+    resp = api_client.get("/api/v1/posts/post-1")
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert data["post"]["uuid"] == "post-1"
+    assert isinstance(data["comments"], list)
+    # The tree has one root with one nested reply.
+    assert data["comments"][0]["uuid"] == "c1"
+    assert data["comments"][0]["replies"][0]["uuid"] == "c2"
+
+
+def test_post_detail_404_for_unknown(api_client):
+    resp = api_client.get("/api/v1/posts/does-not-exist")
+    assert resp.status_code == 404
+
+
+def test_comments_endpoint_returns_tree(api_client):
+    resp = api_client.get("/api/v1/comments/post-1")
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert data["comments"][0]["uuid"] == "c1"
+    assert data["comments"][0]["replies"][0]["uuid"] == "c2"
+
+
+def test_filters_endpoint_lists_filtersets(api_client):
+    resp = api_client.get("/api/v1/filters")
+    assert resp.status_code == 200
+    data = resp.get_json()
+    assert "filters" in data
+    # Stub engine advertises no_filter; the list may be empty or contain it.
+    assert isinstance(data["filters"], list)
+
+
+# Imported via the api_client fixture's monkeypatch; keep the name available.
+from conftest import _StubPolling  # noqa: E402, F401

tests/test_app_factory.py

@@ -0,0 +1,58 @@
+"""App-factory tests: route registration and endpoint names are intact.
+
+These guard the Phase 1 refactor (module-level ``app`` → ``create_app()`` with
+routes split under ``routes/`` and ``blueprints/``). They assert that the
+endpoints referenced by templates via ``url_for(...)`` still resolve in the
+Flask url_map. Runs against in-memory SQLite (no Postgres).
+"""
+
+import pytest
+
+# Endpoints that templates reference via url_for(...) and that must survive
+# the factory refactor. Sourced from the Jinja templates.
+TEMPLATE_ENDPOINTS = [
+    "login",
+    "signup",
+    "admin_setup",
+    "logout",
+    "serve_logo",
+    "serve_theme",
+    "static",
+    # API blueprint endpoints (mounted at /api/v1)
+    "api.posts",
+    "api.post_detail",
+    "api.comments",
+    "api.filters",
+    "api.bookmarks",
+    "api.platforms",
+]
+
+
+def test_create_app_returns_flask_app(app):
+    from flask import Flask
+
+    assert isinstance(app, Flask)
+
+
+def test_api_blueprint_mounted_under_v1(app):
+    rules = [r.rule for r in app.url_map.iter_rules()]
+    assert any(r.startswith("/api/v1/posts") for r in rules), rules
+    assert any(r.startswith("/api/v1/comments") for r in rules), rules
+
+
+@pytest.mark.parametrize("endpoint", TEMPLATE_ENDPOINTS)
+def test_template_endpoints_exist(app, endpoint):
+    # Flask stores endpoints as "<blueprint>.<view>" for blueprint views and
+    # bare names for app-level views. ``url_map`` has both.
+    all_endpoints = {r.endpoint for r in app.url_map.iter_rules()}
+    assert endpoint in all_endpoints, (
+        f"endpoint '{endpoint}' missing from url_map; have: {sorted(all_endpoints)[:20]}..."
+    )
+
+
+def test_no_module_level_app_object():
+    """Phase 1 removed the module-level ``app``; importing app.py must not
+    start a server or expose a global Flask app."""
+    import app as app_module
+
+    assert not hasattr(app_module, "app"), "app.py must not keep a module-level `app`"

tests/test_filter_pipeline.py

@@ -0,0 +1,103 @@
+"""Offline filter-pipeline tests.
+
+These do NOT require Flask or Postgres — only the filter_pipeline package (which
+depends on stdlib + ``requests``). They exercise the registry-driven engine,
+the offline plugin path, AI-disabled fail-open behavior, and comment filtering.
+"""
+
+import pytest
+
+from filter_pipeline.engine import FilterEngine
+from filter_pipeline.models import ProcessingStatus
+from filter_pipeline.registry import (
+    get_registered_plugins,
+    get_registered_stages,
+)
+
+
+@pytest.fixture
+def engine():
+    # Fresh engine (not the singleton) so test isolation holds.
+    return FilterEngine("filter_config.json", "filtersets.json")
+
+
+def test_registry_discovers_builtin_stages_and_plugins(engine):
+    engine._init_stages()
+    stages = get_registered_stages()
+    for name in ["categorizer", "moderator", "filter", "ranker",
+                "plugins", "comment_filter"]:
+        assert name in stages, f"stage '{name}' not registered"
+    plugins = get_registered_plugins()
+    for name in ["keyword", "quality"]:
+        assert name in plugins, f"plugin '{name}' not registered"
+
+
+def test_offline_filterset_runs_without_ai(engine):
+    # quality_filter uses pipeline_stages=['plugins','ranker'] only.
+    posts = [
+        {"uuid": "p1", "title": "A fine Python programming title",
+         "content": "c" * 200, "score": 10, "replies": 2,
+         "platform": "reddit", "source": "python", "timestamp": 1700000000},
+        {"uuid": "p2", "title": "bad", "content": "x", "score": 0, "replies": 0,
+         "platform": "reddit", "source": "python", "timestamp": 1700000000},
+    ]
+    results = engine.process_batch(posts, "quality_filter")
+    assert results[0].passed is True
+    # Quality plugin rejects the 3-char title.
+    assert results[1].passed is False
+    assert any("QualityFilter" in t for t in results[1].tags)
+
+
+def test_ai_filterset_does_not_blank_feed_when_ai_disabled(engine):
+    posts = [{"uuid": "p1", "title": "Hello world this is a fine title",
+              "content": "c" * 200, "score": 10, "replies": 2,
+              "platform": "hackernews", "source": "programming",
+              "timestamp": 1700000000}]
+    out = engine.apply_filterset(posts, "safe_content", use_cache=False)
+    assert len(out) == 1, "AI-disabled filterset must not blank the feed"
+    results = engine.process_batch(posts, "safe_content")
+    assert results[0].status == ProcessingStatus.FAILED
+
+
+def test_no_filter_passes_everything(engine):
+    posts = [{"uuid": "p1", "title": "anything", "content": "", "score": 0,
+              "timestamp": 0}]
+    out = engine.apply_filterset(posts, "no_filter", use_cache=False)
+    assert len(out) == 1
+
+
+def test_comment_filter_individual_mode(engine):
+    comments = [
+        {"uuid": "c1", "content": "this is long enough", "score": 5,
+         "depth": 0, "parent_comment_uuid": None},
+        {"uuid": "c2", "content": "hi", "score": 1, "depth": 1,
+         "parent_comment_uuid": "c1"},
+    ]
+    kept = engine.filter_comments(comments, "quality_filter")
+    assert [c["uuid"] for c in kept] == ["c1"]
+
+
+def test_comment_filter_no_filter_passes_all(engine):
+    comments = [{"uuid": "c1", "content": "hi", "score": 1, "depth": 0,
+                 "parent_comment_uuid": None}]
+    assert len(engine.filter_comments(comments, "no_filter")) == 1
+
+
+def test_comment_filter_unknown_filterset_fails_open(engine):
+    comments = [{"uuid": "c1", "content": "hi", "score": 1, "depth": 0,
+                 "parent_comment_uuid": None}]
+    assert len(engine.filter_comments(comments, "no_such_filterset")) == 1
+
+
+def test_comment_tree_pruning_drops_branch_without_moderation(engine):
+    # safe_content comment rules require moderation.flags.is_safe == True.
+    # With no moderation data attached the field is None -> rule fails closed,
+    # so tree pruning removes the parent and its child.
+    comments = [
+        {"uuid": "r", "content": "root", "score": 5, "depth": 0,
+         "parent_comment_uuid": None},
+        {"uuid": "c", "content": "child", "score": 1, "depth": 1,
+         "parent_comment_uuid": "r"},
+    ]
+    kept = engine.filter_comments(comments, "safe_content")
+    assert kept == []

tests/test_plugin_contract.py

@@ -0,0 +1,90 @@
+"""Plugin/stage contract test (Phase 6).
+
+A drop-in stage and plugin, defined ONLY in this test module via the public
+``@register_stage`` / ``@register_plugin`` decorators, are picked up by the
+engine with zero edits to core files. This is the pluggability guarantee: a
+new filter behavior is a new module + a config entry, never an edit to
+``engine.py``.
+"""
+
+from filter_pipeline.engine import FilterEngine
+from filter_pipeline.models import FilterResult
+from filter_pipeline.plugins.base import BaseFilterPlugin
+from filter_pipeline.registry import (
+    get_plugin_class,
+    get_stage_class,
+    register_plugin,
+    register_stage,
+)
+from filter_pipeline.stages.base_stage import BaseStage
+
+
+@register_stage("sentinel_dropin_stage")
+class SentinelStage(BaseStage):
+    """Drop-in stage that tags any result it sees."""
+
+    def get_name(self):
+        return "Sentinel"
+
+    def process(self, post, result):
+        result.tags.append("sentinel_ran")
+        return result
+
+
+@register_plugin("sentinel_dropin_plugin")
+class SentinelPlugin(BaseFilterPlugin):
+    """Drop-in plugin: never rejects, returns a fixed score."""
+
+    def get_name(self):
+        return "SentinelPlugin"
+
+    def should_filter(self, post, context=None):
+        return False
+
+    def score(self, post, context=None):
+        return 0.9
+
+
+def test_dropin_stage_is_registered():
+    assert get_stage_class("sentinel_dropin_stage") is SentinelStage
+
+
+def test_dropin_plugin_is_registered():
+    assert get_plugin_class("sentinel_dropin_plugin") is SentinelPlugin
+
+
+def test_engine_instantiates_dropin_stage():
+    eng = FilterEngine("filter_config.json", "filtersets.json")
+    eng._init_stages()
+    assert "sentinel_dropin_stage" in eng._stages
+    stage = eng._stages["sentinel_dropin_stage"]
+    # Running the stage through the contract it claims to implement works.
+    result = FilterResult(post_uuid="x", passed=True, score=0.5)
+    out = stage.process({"uuid": "x"}, result)
+    assert "sentinel_ran" in out.tags
+
+
+def test_dropin_stage_can_be_selected_in_a_filterset(tmp_path):
+    """A filterset that lists the drop-in stage actually runs it.
+
+    Builds a throwaway config + filterset on disk so no core file is edited.
+    """
+    import json
+
+    cfg = tmp_path / "cfg.json"
+    cfg.write_text(json.dumps({
+        "ai": {"enabled": False},
+        "cache": {"enabled": False},
+        "pipeline": {"default_stages": ["sentinel_dropin_stage"], "enable_parallel": False},
+        "plugins": {"enabled": [], "configs": {}},
+    }))
+    fs = tmp_path / "fs.json"
+    fs.write_text(json.dumps({"custom": {"post_rules": {}, "comment_rules": {}}}))
+
+    eng = FilterEngine(str(cfg), str(fs))
+    eng._init_stages()
+    results = eng.process_batch(
+        [{"uuid": "p1", "title": "t", "content": "", "score": 0, "timestamp": 0}],
+        "custom",
+    )
+    assert "sentinel_ran" in results[0].tags