Phase 1-3 + 6: pluggable filter system, app factory, test harness

Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md.

Phase 1 — app factory + blueprints + /api/v1:
- app.py -> create_app() factory (no module-level app); entrypoints updated
- routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1
- config.py / extensions.py / security.py extracted; services/ layer added
- endpoint names preserved so template url_for() calls keep resolving
  (static check: all 27 template url_for endpoints are defined routes)

Phase 2 — one pluggable filter system:
- filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules
- engine._init_stages() instantiates registered stages (no hardcoded dict);
  process_batch is AI-aware: only short-circuits to the AI-disabled path when
  a filterset's stages declare requires_ai, so offline filtersets run with AI off
- BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality
  re-enabled via filter_config.json plugins config
- comment tree modes ported to stages/comment_filter.py + shared rules.py;
  wired into /api/v1/posts/<uuid> and /api/v1/comments/<uuid> via
  FilterEngine.filter_comments() (fails open)
- offline quality_filter filterset exercises plugins+ranker without AI
- legacy filter_lib / comment_lib / html_generation_lib / generate_html /
  active_html path deleted

Phase 3 prep — pluggable fetchers + Postgres models:
- Post / Comment SQLAlchemy models added to models.py
- migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run)
- platforms/ fetcher registry (extension point)
- live reads/writes still go through PostService (disk JSON); cutover deferred

Phase 6 — test harness:
- pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres;
  stubbed polling/filter singletons)
- test_app_factory.py (route registration, no module-level app),
  test_api_contracts.py (posts/post_detail/comments/filters shape with
  monkeypatched post_service + get_filter_engine),
  test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated
  locally 12/12 incl. drop-in stage/plugin discovered with zero core edits)

Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements.

Verification: py_compile clean across the project; the Flask-free filter-pipeline
and plugin-contract tests pass locally. App-factory / API-contract tests need
deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated
on docker.

Co-Authored-By: Claude <noreply@anthropic.com>

filter_pipeline/engine.py

@@ -13,6 +13,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
 from .config import FilterConfig
 from .cache import FilterCache
 from .models import FilterResult, ProcessingStatus, AIAnalysisResult
+from .registry import discover_modules, get_registered_stages
 
 logger = logging.getLogger(__name__)
 
@@ -56,24 +57,36 @@ class FilterEngine:
         return cls._instance
 
     def _init_stages(self):
-        """Initialize pipeline stages (lazy loading)"""
+        """Initialize pipeline stages from the registry (lazy loading)."""
         if self._stages is not None:
             return
 
-        from .stages.categorizer import CategorizerStage
-        from .stages.moderator import ModeratorStage
-        from .stages.filter import FilterStage
-        from .stages.ranker import RankerStage
+        # Import built-ins and any configured extension modules for decorator
+        # side effects. This keeps engine orchestration independent of concrete
+        # stage classes and gives plugins a zero-core-edit registration path.
+        discover_modules([
+            'filter_pipeline.stages.categorizer',
+            'filter_pipeline.stages.moderator',
+            'filter_pipeline.stages.filter',
+            'filter_pipeline.stages.ranker',
+            'filter_pipeline.stages.plugins',
+            'filter_pipeline.stages.comment_filter',
+            'filter_pipeline.plugins.keyword',
+            'filter_pipeline.plugins.quality',
+            *self.config.config.get('pipeline', {}).get('stage_modules', []),
+            *self.config.config.get('plugins', {}).get('modules', []),
+        ])
 
-        # Initialize stages based on configuration
         self._stages = {
-            'categorizer': CategorizerStage(self.config, self.cache),
-            'moderator': ModeratorStage(self.config, self.cache),
-            'filter': FilterStage(self.config, self.cache),
-            'ranker': RankerStage(self.config, self.cache)
+            name: stage_cls(self.config, self.cache)
+            for name, stage_cls in get_registered_stages().items()
         }
 
-        logger.info(f"Initialized {len(self._stages)} pipeline stages")
+        logger.info(
+            "Initialized %s registered pipeline stages: %s",
+            len(self._stages),
+            ', '.join(sorted(self._stages.keys()))
+        )
 
     def apply_filterset(
         self,
@@ -180,24 +193,26 @@ class FilterEngine:
         if filterset_name == 'no_filter':
             return self._process_no_filter(posts)
 
-        # Initialize stages if needed
-        if self.config.is_ai_enabled():
-            self._init_stages()
-
-        # If AI is disabled but the filterset requires it, do NOT silently pass
-        # everything as no_filter. Pass the posts through (so the feed is not
-        # blanked) but mark every result as FAILED with an explicit error so the
-        # degradation is observable, not silent.
-        if not self.config.is_ai_enabled() and filterset_name != 'no_filter':
-            logger.warning(
-                f"AI disabled but filterset '{filterset_name}' requires AI - "
-                f"passing posts through unfiltered with FAILED status"
-            )
-            return self._process_ai_disabled(filterset_name, posts)
+        # Initialize stages (registry-driven). This must happen regardless of
+        # whether AI is enabled: offline filtersets (rules/plugins/ranker) still
+        # need their stages instantiated to run.
+        self._init_stages()
 
         # Get pipeline stages for this filterset
         stage_names = self._get_stages_for_filterset(filterset_name)
 
+        # If AI is disabled but the filterset's stages require AI, do NOT silently
+        # pass everything as no_filter. Pass the posts through (so the feed is not
+        # blanked) but mark every result as FAILED with an explicit error so the
+        # degradation is observable, not silent. Filtersets whose stages are all
+        # offline (filter/plugins/ranker/comment_filter) still run normally.
+        if not self.config.is_ai_enabled() and self._stages_need_ai(stage_names):
+            logger.warning(
+                f"AI disabled but filterset '{filterset_name}' requires AI stages "
+                f"({stage_names}) - passing posts through unfiltered with FAILED status"
+            )
+            return self._process_ai_disabled(filterset_name, posts)
+
         # Process posts (parallel or sequential based on config)
         if self.config.is_parallel_enabled():
             results = self._process_batch_parallel(posts, filterset_name, stage_names)
@@ -206,6 +221,16 @@ class FilterEngine:
 
         return results
 
+    def _stages_need_ai(self, stage_names: List[str]) -> bool:
+        """Return True if any named stage class declares ``requires_ai``."""
+        from .registry import get_stage_class
+
+        for name in stage_names:
+            stage_cls = get_stage_class(name)
+            if stage_cls is not None and getattr(stage_cls, 'requires_ai', False):
+                return True
+        return False
+
     def _process_no_filter(self, posts: List[Dict[str, Any]]) -> List[FilterResult]:
         """Process posts with no_filter (all pass with default scores)"""
         results = []
@@ -402,3 +427,27 @@ class FilterEngine:
         self.config.reload()
         self._stages = None  # Force re-initialization of stages
         logger.info("Configuration reloaded")
+
+    def filter_comments(
+        self,
+        comments: List[Dict[str, Any]],
+        filterset_name: str = 'no_filter'
+    ) -> List[Dict[str, Any]]:
+        """Filter a post's flat comment list according to a filterset.
+
+        Comment filtering is tree-shaped (per post) and lives in the registered
+        ``comment_filter`` stage rather than the per-post stage pipeline. The
+        caller (API endpoint) builds the tree from the returned flat list via
+        ``PostService.build_comment_tree``.
+
+        Fails open: if the ``comment_filter`` stage is not registered, the
+        comments are returned unchanged so a missing stage never blanks them.
+        """
+        if not comments:
+            return []
+        self._init_stages()
+        comment_stage = self._stages.get('comment_filter')
+        if not comment_stage:
+            logger.warning("comment_filter stage not registered; returning comments unfiltered")
+            return comments
+        return comment_stage.filter_comments(comments, filterset_name)