chelsea/balanceboard
0
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Phase 1-3 + 6: pluggable filter system, app factory, test harness

Browse Source 
Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md.

Phase 1 — app factory + blueprints + /api/v1:
- app.py -> create_app() factory (no module-level app); entrypoints updated
- routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1
- config.py / extensions.py / security.py extracted; services/ layer added
- endpoint names preserved so template url_for() calls keep resolving
  (static check: all 27 template url_for endpoints are defined routes)

Phase 2 — one pluggable filter system:
- filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules
- engine._init_stages() instantiates registered stages (no hardcoded dict);
  process_batch is AI-aware: only short-circuits to the AI-disabled path when
  a filterset's stages declare requires_ai, so offline filtersets run with AI off
- BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality
  re-enabled via filter_config.json plugins config
- comment tree modes ported to stages/comment_filter.py + shared rules.py;
  wired into /api/v1/posts/<uuid> and /api/v1/comments/<uuid> via
  FilterEngine.filter_comments() (fails open)
- offline quality_filter filterset exercises plugins+ranker without AI
- legacy filter_lib / comment_lib / html_generation_lib / generate_html /
  active_html path deleted

Phase 3 prep — pluggable fetchers + Postgres models:
- Post / Comment SQLAlchemy models added to models.py
- migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run)
- platforms/ fetcher registry (extension point)
- live reads/writes still go through PostService (disk JSON); cutover deferred

Phase 6 — test harness:
- pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres;
  stubbed polling/filter singletons)
- test_app_factory.py (route registration, no module-level app),
  test_api_contracts.py (posts/post_detail/comments/filters shape with
  monkeypatched post_service + get_filter_engine),
  test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated
  locally 12/12 incl. drop-in stage/plugin discovered with zero core edits)

Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements.

Verification: py_compile clean across the project; the Flask-free filter-pipeline
and plugin-contract tests pass locally. App-factory / API-contract tests need
deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated
on docker.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Chelsea
2026-07-03 02:29:46 -05:00
parent cdba720a1c
commit 6cf35ca034
64 changed files with 3902 additions and 3776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
filter_pipeline/__init__.py
View File

@@ -5,6 +5,13 @@ Content filtering, categorization, and ranking system for BalanceBoard.
from .engine import FilterEngine
from .models import FilterResult, ProcessingStatus
from .registry import register_stage, register_plugin
__all__ = ['FilterEngine', 'FilterResult', 'ProcessingStatus']
__version__ = '1.0.0'
__all__ = [
'FilterEngine',
'FilterResult',
'ProcessingStatus',
'register_stage',
'register_plugin',
]
__version__ = '1.0.0'

99
filter_pipeline/engine.py
View File

@@ -13,6 +13,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed
from .config import FilterConfig
from .cache import FilterCache
from .models import FilterResult, ProcessingStatus, AIAnalysisResult
from .registry import discover_modules, get_registered_stages
logger = logging.getLogger(__name__)
@@ -56,24 +57,36 @@ class FilterEngine:
return cls._instance
def _init_stages(self):
"""Initialize pipeline stages (lazy loading)"""
"""Initialize pipeline stages from the registry (lazy loading)."""
if self._stages is not None:
return
from .stages.categorizer import CategorizerStage
from .stages.moderator import ModeratorStage
from .stages.filter import FilterStage
from .stages.ranker import RankerStage
# Import built-ins and any configured extension modules for decorator
# side effects. This keeps engine orchestration independent of concrete
# stage classes and gives plugins a zero-core-edit registration path.
discover_modules([
'filter_pipeline.stages.categorizer',
'filter_pipeline.stages.moderator',
'filter_pipeline.stages.filter',
'filter_pipeline.stages.ranker',
'filter_pipeline.stages.plugins',
'filter_pipeline.stages.comment_filter',
'filter_pipeline.plugins.keyword',
'filter_pipeline.plugins.quality',
*self.config.config.get('pipeline', {}).get('stage_modules', []),
*self.config.config.get('plugins', {}).get('modules', []),
])
# Initialize stages based on configuration
self._stages = {
'categorizer': CategorizerStage(self.config, self.cache),
'moderator': ModeratorStage(self.config, self.cache),
'filter': FilterStage(self.config, self.cache),
'ranker': RankerStage(self.config, self.cache)
name: stage_cls(self.config, self.cache)
for name, stage_cls in get_registered_stages().items()
}
logger.info(f"Initialized {len(self._stages)} pipeline stages")
logger.info(
"Initialized %s registered pipeline stages: %s",
len(self._stages),
', '.join(sorted(self._stages.keys()))
)
def apply_filterset(
self,
@@ -180,24 +193,26 @@ class FilterEngine:
if filterset_name == 'no_filter':
return self._process_no_filter(posts)
# Initialize stages if needed
if self.config.is_ai_enabled():
self._init_stages()
# If AI is disabled but the filterset requires it, do NOT silently pass
# everything as no_filter. Pass the posts through (so the feed is not
# blanked) but mark every result as FAILED with an explicit error so the
# degradation is observable, not silent.
if not self.config.is_ai_enabled() and filterset_name != 'no_filter':
logger.warning(
f"AI disabled but filterset '{filterset_name}' requires AI - "
f"passing posts through unfiltered with FAILED status"
)
return self._process_ai_disabled(filterset_name, posts)
# Initialize stages (registry-driven). This must happen regardless of
# whether AI is enabled: offline filtersets (rules/plugins/ranker) still
# need their stages instantiated to run.
self._init_stages()
# Get pipeline stages for this filterset
stage_names = self._get_stages_for_filterset(filterset_name)
# If AI is disabled but the filterset's stages require AI, do NOT silently
# pass everything as no_filter. Pass the posts through (so the feed is not
# blanked) but mark every result as FAILED with an explicit error so the
# degradation is observable, not silent. Filtersets whose stages are all
# offline (filter/plugins/ranker/comment_filter) still run normally.
if not self.config.is_ai_enabled() and self._stages_need_ai(stage_names):
logger.warning(
f"AI disabled but filterset '{filterset_name}' requires AI stages "
f"({stage_names}) - passing posts through unfiltered with FAILED status"
)
return self._process_ai_disabled(filterset_name, posts)
# Process posts (parallel or sequential based on config)
if self.config.is_parallel_enabled():
results = self._process_batch_parallel(posts, filterset_name, stage_names)
@@ -206,6 +221,16 @@ class FilterEngine:
return results
def _stages_need_ai(self, stage_names: List[str]) -> bool:
"""Return True if any named stage class declares ``requires_ai``."""
from .registry import get_stage_class
for name in stage_names:
stage_cls = get_stage_class(name)
if stage_cls is not None and getattr(stage_cls, 'requires_ai', False):
return True
return False
def _process_no_filter(self, posts: List[Dict[str, Any]]) -> List[FilterResult]:
"""Process posts with no_filter (all pass with default scores)"""
results = []
@@ -402,3 +427,27 @@ class FilterEngine:
self.config.reload()
self._stages = None # Force re-initialization of stages
logger.info("Configuration reloaded")
def filter_comments(
self,
comments: List[Dict[str, Any]],
filterset_name: str = 'no_filter'
) -> List[Dict[str, Any]]:
"""Filter a post's flat comment list according to a filterset.
Comment filtering is tree-shaped (per post) and lives in the registered
``comment_filter`` stage rather than the per-post stage pipeline. The
caller (API endpoint) builds the tree from the returned flat list via
``PostService.build_comment_tree``.
Fails open: if the ``comment_filter`` stage is not registered, the
comments are returned unchanged so a missing stage never blanks them.
"""
if not comments:
return []
self._init_stages()
comment_stage = self._stages.get('comment_filter')
if not comment_stage:
logger.warning("comment_filter stage not registered; returning comments unfiltered")
return comments
return comment_stage.filter_comments(comments, filterset_name)

2
filter_pipeline/plugins/keyword.py
View File

@@ -7,10 +7,12 @@ import logging
from typing import Dict, Any, Optional, List
from .base import BaseFilterPlugin
from ..registry import register_plugin
logger = logging.getLogger(__name__)
@register_plugin("keyword")
class KeywordFilterPlugin(BaseFilterPlugin):
"""
Filter posts based on keyword matching.

2
filter_pipeline/plugins/quality.py
View File

@@ -8,10 +8,12 @@ import re
from typing import Dict, Any, Optional
from .base import BaseFilterPlugin
from ..registry import register_plugin
logger = logging.getLogger(__name__)
@register_plugin("quality")
class QualityFilterPlugin(BaseFilterPlugin):
"""
Filter posts based on quality metrics.

62
filter_pipeline/registry.py Normal file
View File

@@ -0,0 +1,62 @@
"""Registries for filter pipeline stages and plugins."""
import importlib
import logging
from typing import Any, Dict, Iterable, Optional, Type
logger = logging.getLogger(__name__)
_STAGE_REGISTRY: Dict[str, Type[Any]] = {}
_PLUGIN_REGISTRY: Dict[str, Type[Any]] = {}
_DISCOVERED_MODULES = set()
def register_stage(name: str):
"""Register a pipeline stage class under a config name."""
def decorator(stage_cls: Type[Any]):
if name in _STAGE_REGISTRY and _STAGE_REGISTRY[name] is not stage_cls:
logger.warning("Replacing registered filter stage '%s'", name)
_STAGE_REGISTRY[name] = stage_cls
return stage_cls
return decorator
def register_plugin(name: str):
"""Register a filter plugin class under a config name."""
def decorator(plugin_cls: Type[Any]):
if name in _PLUGIN_REGISTRY and _PLUGIN_REGISTRY[name] is not plugin_cls:
logger.warning("Replacing registered filter plugin '%s'", name)
_PLUGIN_REGISTRY[name] = plugin_cls
return plugin_cls
return decorator
def get_stage_class(name: str) -> Optional[Type[Any]]:
"""Return a registered stage class by name."""
return _STAGE_REGISTRY.get(name)
def get_plugin_class(name: str) -> Optional[Type[Any]]:
"""Return a registered plugin class by name."""
return _PLUGIN_REGISTRY.get(name)
def get_registered_stages() -> Dict[str, Type[Any]]:
"""Return a copy of registered stage classes."""
return dict(_STAGE_REGISTRY)
def get_registered_plugins() -> Dict[str, Type[Any]]:
"""Return a copy of registered plugin classes."""
return dict(_PLUGIN_REGISTRY)
def discover_modules(module_names: Iterable[str]):
"""Import modules for registration side effects once."""
for module_name in module_names:
if not module_name or module_name in _DISCOVERED_MODULES:
continue
importlib.import_module(module_name)
_DISCOVERED_MODULES.add(module_name)

80
filter_pipeline/rules.py Normal file
View File

@@ -0,0 +1,80 @@
"""Shared rule evaluation for posts and comments."""
from typing import Any, Dict
def get_nested_value(obj: Dict[str, Any], path: str) -> Any:
"""Get a nested dict value using dot notation."""
value = obj
for key in path.split("."):
if isinstance(value, dict) and key in value:
value = value[key]
else:
return None
return value
def evaluate_rule(value: Any, operator: str, target: Any) -> bool:
"""Evaluate one rule operator."""
if value is None:
return False
if operator == "equals":
return value == target
if operator == "not_equals":
return value != target
if operator == "in":
return value in target
if operator == "not_in":
return value not in target
if operator == "min":
return value >= target
if operator == "max":
return value <= target
if operator == "after":
return value > target
if operator == "before":
return value < target
if operator == "contains":
return target in value
if operator == "excludes":
if isinstance(value, list):
return not any(item in target for item in value)
return value not in target
if operator == "includes":
if isinstance(value, list):
return target in value
return False
if operator == "includes_any":
if isinstance(value, list) and isinstance(target, list):
for item in value:
if isinstance(item, dict):
for rule in target:
if (
isinstance(rule, dict)
and item.get("topic") == rule.get("topic")
and item.get("confidence", 0) >= rule.get("confidence_min", 0)
):
return True
elif item in target:
return True
return False
if operator == "min_length":
return len(str(value)) >= target
if operator == "max_length":
return len(str(value)) <= target
return False
def apply_rules(item: Dict[str, Any], rules: Dict[str, Dict[str, Any]]) -> bool:
"""Return True when all field rules pass."""
if not rules:
return True
for field_path, rule_def in rules.items():
value = get_nested_value(item, field_path)
for operator, target in rule_def.items():
if not evaluate_rule(value, operator, target):
return False
return True

12
filter_pipeline/stages/__init__.py
View File

@@ -8,5 +8,15 @@ from .categorizer import CategorizerStage
from .moderator import ModeratorStage
from .filter import FilterStage
from .ranker import RankerStage
from .plugins import PluginStage
from .comment_filter import CommentFilterStage
__all__ = ['BaseStage', 'CategorizerStage', 'ModeratorStage', 'FilterStage', 'RankerStage']
__all__ = [
'BaseStage',
'CategorizerStage',
'ModeratorStage',
'FilterStage',
'RankerStage',
'PluginStage',
'CommentFilterStage',
]

16
filter_pipeline/stages/base_stage.py
View File

@@ -14,19 +14,29 @@ class BaseStage(ABC):
Each stage processes posts sequentially and can modify FilterResults.
Stages are executed in order: Categorizer → Moderator → Filter → Ranker
``requires_ai`` marks stages that need the AI client. The engine uses it
to decide whether a filterset can run with AI disabled (offline filtersets
that only use rule/plugin/ranker stages still run; AI stages short-circuit
to the AI-disabled path so the feed is not silently blanked).
"""
def __init__(self, config: Dict[str, Any], cache: Any):
requires_ai: bool = False
def __init__(self, config: "FilterConfig", cache: Any):
"""
Initialize stage.
Args:
config: Configuration dictionary for this stage
config: FilterConfig instance for this pipeline run
cache: FilterCache instance
"""
self.config = config
self.cache = cache
self.enabled = config.get('enabled', True)
# Stages are enabled by default; a per-stage enabled flag can be set
# by subclasses reading their own config section. FilterConfig is not a
# dict, so do not call ``config.get(...)`` here.
self.enabled = True
@abstractmethod
def process(

4
filter_pipeline/stages/categorizer.py
View File

@@ -8,6 +8,7 @@ from typing import Dict, Any
from datetime import datetime
from .base_stage import BaseStage
from ..registry import register_stage
from ..models import FilterResult, AIAnalysisResult
from ..cache import FilterCache
from ..ai_client import OpenRouterClient
@@ -15,6 +16,7 @@ from ..ai_client import OpenRouterClient
logger = logging.getLogger(__name__)
@register_stage("categorizer")
class CategorizerStage(BaseStage):
"""
Stage 1: Categorize content and extract tags.
@@ -22,6 +24,8 @@ class CategorizerStage(BaseStage):
Uses AI to detect topics/categories with content-hash based caching.
"""
requires_ai = True
def __init__(self, config, cache: FilterCache):
super().__init__(config, cache)

124
filter_pipeline/stages/comment_filter.py Normal file
View File

@@ -0,0 +1,124 @@
"""Comment filtering stage and tree modes."""
import logging
from datetime import datetime
from typing import Any, Dict, List
from .base_stage import BaseStage
from ..models import FilterResult
from ..registry import register_stage
from ..rules import apply_rules
logger = logging.getLogger(__name__)
@register_stage("comment_filter")
class CommentFilterStage(BaseStage):
"""Apply filterset comment rules using configured tree modes."""
def get_name(self) -> str:
return "CommentFilter"
def process(self, post: Dict[str, Any], result: FilterResult) -> FilterResult:
"""Post pipeline no-op; comments are filtered through filter_comments()."""
return result
def filter_comments(self, comments: List[Dict[str, Any]], filterset_name: str) -> List[Dict[str, Any]]:
if not comments:
return []
filterset = self.config.get_filterset(filterset_name) or {}
rules = filterset.get("comment_rules", {})
mode = filterset.get("comment_filter_mode", "individual")
if not rules:
return [dict(comment) for comment in comments]
if mode == "tree_pruning":
return self._filter_tree_pruning(comments, rules)
if mode == "score_based":
return self._filter_individual(comments, rules, extra_check=self._passes_score_rules)
if mode == "time_bound":
return self._filter_individual(comments, rules, extra_check=self._passes_time_rules)
if mode == "content_length":
return self._filter_individual(comments, rules, extra_check=self._passes_length_rules)
return self._filter_individual(comments, rules)
def _filter_tree_pruning(self, comments: List[Dict[str, Any]], rules: Dict[str, Any]) -> List[Dict[str, Any]]:
comment_map = {comment["uuid"]: {**comment, "children": []} for comment in comments if comment.get("uuid")}
roots = []
for comment in comments:
uuid = comment.get("uuid")
if not uuid or uuid not in comment_map:
continue
parent_uuid = comment.get("parent_comment_uuid")
if parent_uuid and parent_uuid in comment_map:
comment_map[parent_uuid]["children"].append(comment_map[uuid])
else:
roots.append(comment_map[uuid])
def prune(nodes):
pruned = []
for node in nodes:
if self._passes_comment_rules(node, rules):
node["children"] = prune(node.get("children", []))
pruned.append(node)
return pruned
return self._flatten_tree(prune(roots))
def _filter_individual(self, comments, rules, extra_check=None):
filtered = []
for comment in comments:
item = dict(comment)
if self._passes_comment_rules(item, rules) and (extra_check is None or extra_check(item, rules)):
filtered.append(item)
return filtered
def _passes_comment_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool:
return apply_rules(comment, rules)
def _passes_score_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool:
score_rules = rules.get("score", {})
min_score = score_rules.get("min", -1000)
return comment.get("score", 0) >= min_score
def _passes_time_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool:
time_rules = rules.get("timestamp", {})
timestamp = comment.get("timestamp")
if not timestamp:
return "timestamp" not in rules
try:
if isinstance(timestamp, (int, float)):
comment_time = datetime.fromtimestamp(timestamp)
else:
comment_time = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00"))
after = time_rules.get("after")
before = time_rules.get("before")
if after and comment_time <= datetime.fromisoformat(str(after).replace("Z", "+00:00")):
return False
if before and comment_time >= datetime.fromisoformat(str(before).replace("Z", "+00:00")):
return False
return True
except (ValueError, TypeError):
return False
def _passes_length_rules(self, comment: Dict[str, Any], rules: Dict[str, Any]) -> bool:
length_rules = rules.get("content_length", {})
content_length = len(comment.get("content", ""))
min_length = length_rules.get("min", 0)
max_length = length_rules.get("max", float("inf"))
return min_length <= content_length <= max_length
def _flatten_tree(self, tree: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
flat = []
def traverse(nodes):
for node in nodes:
children = node.pop("children", [])
flat.append(node)
traverse(children)
traverse(tree)
return flat

147
filter_pipeline/stages/filter.py
View File

@@ -4,14 +4,17 @@ Apply filterset rules to posts (no AI needed - fast rule evaluation).
"""
import logging
from typing import Dict, Any, List
from typing import Dict, Any
from .base_stage import BaseStage
from ..models import FilterResult
from ..registry import register_stage
from ..rules import apply_rules
logger = logging.getLogger(__name__)
@register_stage("filter")
class FilterStage(BaseStage):
"""
Stage 3: Apply filterset rules.
@@ -24,148 +27,20 @@ class FilterStage(BaseStage):
return "Filter"
def process(self, post: Dict[str, Any], result: FilterResult) -> FilterResult:
"""
Apply filterset rules to post.
Args:
post: Post data
result: Current FilterResult
Returns:
Updated FilterResult (may be rejected)
"""
# Get filterset configuration
"""Apply filterset post rules to a post."""
filterset = self.config.get_filterset(result.filterset_name)
if not filterset:
logger.warning(f"Filterset '{result.filterset_name}' not found")
return result
# Apply post rules
post_rules = filterset.get('post_rules', {})
if not self._evaluate_rules(post, result, post_rules):
item = dict(post)
if result.moderation_data:
item["moderation"] = result.moderation_data
if not apply_rules(item, filterset.get("post_rules", {})):
result.passed = False
logger.debug(f"Filter: Post {post.get('uuid', '')} rejected by filterset rules")
return result
# Post passed all rules
logger.debug(f"Filter: Post {post.get('uuid', '')} passed filterset '{result.filterset_name}'")
return result
def _evaluate_rules(
self,
post: Dict[str, Any],
result: FilterResult,
rules: Dict[str, Any]
) -> bool:
"""
Evaluate all rules for a post.
Returns:
True if post passes all rules, False otherwise
"""
for field, condition in rules.items():
if not self._evaluate_condition(post, result, field, condition):
logger.debug(f"Filter: Failed condition '{field}': {condition}")
return False
return True
def _evaluate_condition(
self,
post: Dict[str, Any],
result: FilterResult,
field: str,
condition: Any
) -> bool:
"""
Evaluate a single condition.
Supported conditions:
- {"equals": value}
- {"not_equals": value}
- {"in": [values]}
- {"not_in": [values]}
- {"min": value}
- {"max": value}
- {"includes_any": [values]}
- {"excludes": [values]}
Args:
post: Post data
result: FilterResult with moderation data
field: Field path (e.g., "score", "moderation.flags.is_safe")
condition: Condition dict
Returns:
True if condition passes
"""
# Get field value
value = self._get_field_value(post, result, field)
# Evaluate condition
if isinstance(condition, dict):
for op, expected in condition.items():
if op == 'equals':
if value != expected:
return False
elif op == 'not_equals':
if value == expected:
return False
elif op == 'in':
if value not in expected:
return False
elif op == 'not_in':
if value in expected:
return False
elif op == 'min':
if value < expected:
return False
elif op == 'max':
if value > expected:
return False
elif op == 'includes_any':
# Check if any expected value is in the field (for lists)
if not isinstance(value, list):
return False
if not any(item in value for item in expected):
return False
elif op == 'excludes':
# Check that none of the excluded values are present
if isinstance(value, list):
if any(item in expected for item in value):
return False
elif value in expected:
return False
else:
logger.warning(f"Unknown condition operator: {op}")
return True
def _get_field_value(self, post: Dict[str, Any], result: FilterResult, field: str):
"""
Get field value from post or result.
Supports nested fields like "moderation.flags.is_safe"
"""
parts = field.split('.')
# Check if field is in moderation data
if parts[0] == 'moderation' and result.moderation_data:
value = result.moderation_data
for part in parts[1:]:
if isinstance(value, dict):
value = value.get(part)
else:
return None
return value
# Check post data
value = post
for part in parts:
if isinstance(value, dict):
value = value.get(part)
else:
return None
return value
return result

4
filter_pipeline/stages/moderator.py
View File

@@ -8,6 +8,7 @@ from typing import Dict, Any
from datetime import datetime
from .base_stage import BaseStage
from ..registry import register_stage
from ..models import FilterResult, AIAnalysisResult
from ..cache import FilterCache
from ..ai_client import OpenRouterClient
@@ -15,6 +16,7 @@ from ..ai_client import OpenRouterClient
logger = logging.getLogger(__name__)
@register_stage("moderator")
class ModeratorStage(BaseStage):
"""
Stage 2: Content moderation and quality analysis.
@@ -22,6 +24,8 @@ class ModeratorStage(BaseStage):
Uses AI to analyze safety, quality, and sentiment with content-hash based caching.
"""
requires_ai = True
def __init__(self, config, cache: FilterCache):
super().__init__(config, cache)

89
filter_pipeline/stages/plugins.py Normal file
View File

@@ -0,0 +1,89 @@
"""Plugin consumer stage for registered BaseFilterPlugin implementations."""
import logging
from typing import Any, Dict, List
from .base_stage import BaseStage
from ..models import FilterResult
from ..registry import get_plugin_class, register_stage
logger = logging.getLogger(__name__)
@register_stage("plugins")
class PluginStage(BaseStage):
"""Run configured filter plugins against each post."""
def __init__(self, config, cache):
super().__init__(config, cache)
self._plugin_instances = None
def get_name(self) -> str:
return "Plugins"
def process(self, post: Dict[str, Any], result: FilterResult) -> FilterResult:
"""Apply configured plugins to a post/result pair."""
plugins = self._get_plugins(result.filterset_name)
if not plugins:
return result
context = {
"filterset_name": result.filterset_name,
"categories": result.categories,
"tags": result.tags,
"moderation": result.moderation_data,
"score_breakdown": result.score_breakdown,
}
plugin_scores = []
for plugin in plugins:
if not plugin.is_enabled():
continue
try:
if plugin.should_filter(post, context):
result.passed = False
result.tags.append(f"plugin:{plugin.get_name()}:rejected")
logger.debug("Plugin %s rejected post %s", plugin.get_name(), post.get("uuid", ""))
return result
score = plugin.score(post, context)
plugin_scores.append(score)
result.score_breakdown[f"plugin:{plugin.get_name()}"] = score
result.tags.append(f"plugin:{plugin.get_name()}")
except Exception as e:
logger.error("Plugin %s failed: %s", plugin.get_name(), e)
result.error = f"plugin:{plugin.get_name()}: {e}"
result.passed = False
return result
if plugin_scores:
result.score_breakdown["plugins"] = sum(plugin_scores) / len(plugin_scores)
# Blend plugin judgment with the current score without replacing
# ranking completely. Ranker can still run later and overwrite the
# final score from its own weighted factors.
result.score = (result.score + result.score_breakdown["plugins"]) / 2
return result
def _get_plugins(self, filterset_name: str) -> List[Any]:
if self._plugin_instances is None:
self._plugin_instances = self._build_plugin_instances()
filterset = self.config.get_filterset(filterset_name) or {}
plugin_names = filterset.get("plugins")
if plugin_names is None:
plugin_names = self.config.config.get("plugins", {}).get("enabled", [])
return [self._plugin_instances[name] for name in plugin_names if name in self._plugin_instances]
def _build_plugin_instances(self) -> Dict[str, Any]:
plugin_config = self.config.config.get("plugins", {})
instances = {}
for name, settings in plugin_config.get("configs", {}).items():
plugin_cls = get_plugin_class(name)
if not plugin_cls:
logger.warning("Configured plugin '%s' is not registered", name)
continue
instances[name] = plugin_cls(settings or {})
return instances

2
filter_pipeline/stages/ranker.py
View File

@@ -8,11 +8,13 @@ from typing import Dict, Any
from datetime import datetime
from .base_stage import BaseStage
from ..registry import register_stage
from ..models import FilterResult
logger = logging.getLogger(__name__)
@register_stage("ranker")
class RankerStage(BaseStage):
"""
Stage 4: Score and rank posts.