Files
Chelsea 6cf35ca034 Phase 1-3 + 6: pluggable filter system, app factory, test harness
Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md.

Phase 1 — app factory + blueprints + /api/v1:
- app.py -> create_app() factory (no module-level app); entrypoints updated
- routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1
- config.py / extensions.py / security.py extracted; services/ layer added
- endpoint names preserved so template url_for() calls keep resolving
  (static check: all 27 template url_for endpoints are defined routes)

Phase 2 — one pluggable filter system:
- filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules
- engine._init_stages() instantiates registered stages (no hardcoded dict);
  process_batch is AI-aware: only short-circuits to the AI-disabled path when
  a filterset's stages declare requires_ai, so offline filtersets run with AI off
- BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality
  re-enabled via filter_config.json plugins config
- comment tree modes ported to stages/comment_filter.py + shared rules.py;
  wired into /api/v1/posts/<uuid> and /api/v1/comments/<uuid> via
  FilterEngine.filter_comments() (fails open)
- offline quality_filter filterset exercises plugins+ranker without AI
- legacy filter_lib / comment_lib / html_generation_lib / generate_html /
  active_html path deleted

Phase 3 prep — pluggable fetchers + Postgres models:
- Post / Comment SQLAlchemy models added to models.py
- migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run)
- platforms/ fetcher registry (extension point)
- live reads/writes still go through PostService (disk JSON); cutover deferred

Phase 6 — test harness:
- pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres;
  stubbed polling/filter singletons)
- test_app_factory.py (route registration, no module-level app),
  test_api_contracts.py (posts/post_detail/comments/filters shape with
  monkeypatched post_service + get_filter_engine),
  test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated
  locally 12/12 incl. drop-in stage/plugin discovered with zero core edits)

Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements.

Verification: py_compile clean across the project; the Flask-free filter-pipeline
and plugin-contract tests pass locally. App-factory / API-contract tests need
deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated
on docker.

Co-Authored-By: Claude <noreply@anthropic.com>
2026-07-03 02:29:46 -05:00

464 lines
20 KiB
Python

"""Versioned JSON API blueprint."""
import json
import logging
from datetime import datetime, timedelta
from flask import Blueprint, jsonify, request
from flask_login import current_user, login_required
from config import DEFAULT_PAGE_SIZE
from database import db
from extensions import get_filter_engine
from models import Bookmark
from security import is_safe_filterset
from services import get_display_name_for_source, load_platform_config, post_service
logger = logging.getLogger(__name__)
FILTER_ICON_MAP = {
"no_filter": "\U0001f310",
"safe_content": "\u2705",
"tech_only": "\U0001f4bb",
"high_quality": "\u2b50",
"custom_example": "\U0001f3af",
}
def _current_filterset():
"""Return selected filterset from query override or current user settings."""
filter_override = request.args.get("filter", "")
if filter_override and is_safe_filterset(filter_override):
return filter_override
if current_user.is_authenticated:
try:
user_settings = json.loads(current_user.settings) if current_user.settings else {}
return user_settings.get("filter_set", "no_filter")
except (json.JSONDecodeError, TypeError):
pass
return "no_filter"
FILTER_NAME_MAP = {
"no_filter": "All Content",
"safe_content": "Safe Content",
"tech_only": "Tech Only",
"high_quality": "High Quality",
"custom_example": "Custom Example",
}
def create_api_blueprint(name="api"):
"""Create the API blueprint so it can be mounted with a version prefix."""
bp = Blueprint(name, __name__)
@bp.get("/posts")
def posts():
"""Get paginated posts with filtering."""
try:
platform_config = load_platform_config()
page = int(request.args.get("page", 1))
per_page = int(request.args.get("per_page", DEFAULT_PAGE_SIZE))
community = request.args.get("community", "")
platform = request.args.get("platform", "")
search_query = request.args.get("q", "").lower().strip()
filter_override = request.args.get("filter", "")
filterset_name = "no_filter"
user_communities = []
time_filter_enabled = False
time_filter_days = 7
if current_user.is_authenticated:
try:
user_settings = json.loads(current_user.settings) if current_user.settings else {}
filterset_name = user_settings.get("filter_set", "no_filter")
user_communities = user_settings.get("communities", [])
experience_settings = user_settings.get("experience", {})
time_filter_enabled = experience_settings.get("time_filter_enabled", False)
time_filter_days = experience_settings.get("time_filter_days", 7)
except (json.JSONDecodeError, TypeError):
filterset_name = "no_filter"
user_communities = []
time_filter_enabled = False
time_filter_days = 7
if filter_override and is_safe_filterset(filter_override):
filterset_name = filter_override
cached_posts, cached_comments = post_service.load()
time_cutoff = None
if time_filter_enabled:
cutoff_date = datetime.utcnow() - timedelta(days=time_filter_days)
time_cutoff = cutoff_date.timestamp()
raw_posts = []
for post_data in cached_posts.values():
if time_filter_enabled and time_cutoff:
post_timestamp = post_data.get("timestamp", 0)
if post_timestamp < time_cutoff:
continue
if community and post_data.get("source", "").lower() != community.lower():
continue
if platform and post_data.get("platform", "").lower() != platform.lower():
continue
if user_communities:
post_source = post_data.get("source", "").lower()
post_platform = post_data.get("platform", "").lower()
post_id = post_data.get("id", "").lower()
matches_community = any(
post_source == selected.lower()
or post_platform == selected.lower()
or selected.lower() in post_source
or selected.lower() in post_id
for selected in user_communities
if isinstance(selected, str)
)
if not matches_community:
continue
if search_query:
title = post_data.get("title", "").lower()
content = post_data.get("content", "").lower()
author = post_data.get("author", "").lower()
source = post_data.get("source", "").lower()
if not (
search_query in title
or search_query in content
or search_query in author
or search_query in source
):
continue
raw_posts.append(post_data)
filtered_posts = get_filter_engine().apply_filterset(
raw_posts, filterset_name, use_cache=True
)
response_posts = []
for post_data in filtered_posts:
post_uuid = post_data.get("uuid")
source_display = get_display_name_for_source(
post_data.get("platform", ""),
post_data.get("source", ""),
platform_config,
)
response_posts.append(
{
"id": post_uuid,
"title": post_data.get("title", "Untitled"),
"author": post_data.get("author", "Unknown"),
"platform": post_data.get("platform", "unknown"),
"score": post_data.get("score", 0),
"timestamp": post_data.get("timestamp", 0),
"url": f"/post/{post_uuid}",
"comments_count": len(cached_comments.get(post_uuid, [])),
"content_preview": (post_data.get("content", "") or "")[:200] + "..." if post_data.get("content") else "",
"source": post_data.get("source", ""),
"source_display": source_display,
"tags": post_data.get("tags", []),
"external_url": post_data.get("url", ""),
"filter_score": post_data.get("_filter_score", 0.5),
"filter_categories": post_data.get("_filter_categories", []),
"filter_tags": post_data.get("_filter_tags", []),
}
)
response_posts.sort(key=lambda x: (x["filter_score"], x["timestamp"]), reverse=True)
total_posts = len(response_posts)
start_idx = (page - 1) * per_page
end_idx = start_idx + per_page
total_pages = (total_posts + per_page - 1) // per_page
return jsonify(
{
"posts": response_posts[start_idx:end_idx],
"pagination": {
"current_page": page,
"total_pages": total_pages,
"total_posts": total_posts,
"per_page": per_page,
"has_next": page < total_pages,
"has_prev": page > 1,
},
}
)
except Exception as e:
logger.error(f"Error loading posts: {e}")
return jsonify(
{
"posts": [],
"error": str(e),
"pagination": {
"current_page": 1,
"total_pages": 0,
"total_posts": 0,
"per_page": DEFAULT_PAGE_SIZE,
"has_next": False,
"has_prev": False,
},
}
)
@bp.get("/posts/<post_uuid>")
def post_detail(post_uuid):
"""Get one post with its comment tree."""
try:
platform_config = load_platform_config()
cached_posts, cached_comments = post_service.load()
post_data = cached_posts.get(post_uuid)
if not post_data:
return jsonify({"error": "Post not found"}), 404
post = dict(post_data)
post["source_display"] = get_display_name_for_source(
post.get("platform", ""), post.get("source", ""), platform_config
)
filterset_name = _current_filterset()
filtered_comments = get_filter_engine().filter_comments(
cached_comments.get(post_uuid, []), filterset_name
)
comments = post_service.build_comment_tree(filtered_comments)
return jsonify({"post": post, "comments": comments})
except Exception as e:
logger.error(f"Error loading post {post_uuid}: {e}")
return jsonify({"error": "Failed to load post"}), 500
@bp.get("/comments/<post_uuid>")
def comments(post_uuid):
"""Get comments for a post as a tree."""
try:
_, cached_comments = post_service.load()
filterset_name = _current_filterset()
filtered_comments = get_filter_engine().filter_comments(
cached_comments.get(post_uuid, []), filterset_name
)
return jsonify({"comments": post_service.build_comment_tree(filtered_comments)})
except Exception as e:
logger.error(f"Error loading comments for {post_uuid}: {e}")
return jsonify({"error": "Failed to load comments"}), 500
@bp.get("/platforms")
def platforms():
"""Get platform configuration and available communities."""
try:
platform_config = load_platform_config()
communities = []
for key, count in post_service.source_counts().items():
platform, source = key.split(":", 1)
platform_info = platform_config.get("platforms", {}).get(platform, {})
community_info = None
if platform_info.get("supports_communities"):
for community in platform_info.get("communities", []):
if community["id"] == source:
community_info = community
break
if community_info:
communities.append(
{
"platform": platform,
"id": source,
"name": community_info["name"],
"display_name": community_info["display_name"],
"icon": community_info.get("icon", platform_info.get("icon", "\U0001f4c4")),
"count": count,
"description": community_info.get("description", ""),
}
)
else:
display_name = get_display_name_for_source(platform, source, platform_config)
communities.append(
{
"platform": platform,
"id": source,
"name": source or platform,
"display_name": display_name,
"icon": platform_info.get("icon", "\U0001f4c4"),
"count": count,
"description": f"Posts from {display_name}",
}
)
communities.sort(key=lambda x: x["count"], reverse=True)
return jsonify(
{
"platforms": platform_config.get("platforms", {}),
"communities": communities,
"total_communities": len(communities),
}
)
except Exception as e:
logger.error(f"Error loading platform configuration: {e}")
return jsonify({"platforms": {}, "communities": [], "total_communities": 0, "error": str(e)})
@bp.get("/content-timestamp")
def content_timestamp():
"""Get the last content update timestamp for auto-refresh."""
try:
return jsonify({"timestamp": post_service.latest_content_mtime()})
except Exception as e:
logger.error(f"Error getting content timestamp: {e}")
return jsonify({"error": "Failed to get content timestamp"}), 500
@bp.post("/bookmark")
@login_required
def bookmark():
"""Toggle bookmark status for a post."""
try:
data = request.get_json()
if not data or "post_uuid" not in data:
return jsonify({"error": "Missing post_uuid"}), 400
post_uuid = data["post_uuid"]
if not post_uuid:
return jsonify({"error": "Invalid post_uuid"}), 400
existing_bookmark = Bookmark.query.filter_by(
user_id=current_user.id, post_uuid=post_uuid
).first()
if existing_bookmark:
db.session.delete(existing_bookmark)
db.session.commit()
return jsonify({"bookmarked": False, "message": "Bookmark removed"})
cached_posts, _ = post_service.load()
post_data = cached_posts.get(post_uuid, {})
new_bookmark = Bookmark(
user_id=current_user.id,
post_uuid=post_uuid,
title=post_data.get("title", ""),
platform=post_data.get("platform", ""),
source=post_data.get("source", ""),
)
db.session.add(new_bookmark)
db.session.commit()
return jsonify({"bookmarked": True, "message": "Bookmark added"})
except Exception as e:
db.session.rollback()
logger.error(f"Error toggling bookmark: {e}")
return jsonify({"error": "Failed to toggle bookmark"}), 500
@bp.get("/bookmarks")
@login_required
def bookmarks():
"""Get the current user's bookmarks."""
try:
page = int(request.args.get("page", 1))
per_page = int(request.args.get("per_page", DEFAULT_PAGE_SIZE))
bookmarks_query = Bookmark.query.filter_by(user_id=current_user.id).order_by(
Bookmark.created_at.desc()
)
total_bookmarks = bookmarks_query.count()
bookmark_rows = bookmarks_query.offset((page - 1) * per_page).limit(per_page).all()
cached_posts, cached_comments = post_service.load()
bookmark_posts = []
for bookmark_row in bookmark_rows:
post_data = cached_posts.get(bookmark_row.post_uuid)
if post_data:
bookmark_posts.append(
{
"id": bookmark_row.post_uuid,
"title": post_data.get("title", bookmark_row.title or "Untitled"),
"author": post_data.get("author", "Unknown"),
"platform": post_data.get("platform", bookmark_row.platform or "unknown"),
"score": post_data.get("score", 0),
"timestamp": post_data.get("timestamp", 0),
"url": f"/post/{bookmark_row.post_uuid}",
"comments_count": len(cached_comments.get(bookmark_row.post_uuid, [])),
"content_preview": (post_data.get("content", "") or "")[:200] + "..." if post_data.get("content") else "",
"source": post_data.get("source", bookmark_row.source or ""),
"bookmarked_at": bookmark_row.created_at.isoformat(),
"external_url": post_data.get("url", ""),
}
)
else:
bookmark_posts.append(
{
"id": bookmark_row.post_uuid,
"title": bookmark_row.title or "Untitled",
"author": "Unknown",
"platform": bookmark_row.platform or "unknown",
"score": 0,
"timestamp": 0,
"url": f"/post/{bookmark_row.post_uuid}",
"comments_count": 0,
"content_preview": "Content no longer available",
"source": bookmark_row.source or "",
"bookmarked_at": bookmark_row.created_at.isoformat(),
"external_url": "",
"archived": True,
}
)
total_pages = (total_bookmarks + per_page - 1) // per_page
return jsonify(
{
"posts": bookmark_posts,
"pagination": {
"current_page": page,
"total_pages": total_pages,
"total_posts": total_bookmarks,
"per_page": per_page,
"has_next": page < total_pages,
"has_prev": page > 1,
},
}
)
except Exception as e:
logger.error(f"Error getting bookmarks: {e}")
return jsonify({"error": "Failed to get bookmarks"}), 500
@bp.get("/bookmark-status/<post_uuid>")
@login_required
def bookmark_status(post_uuid):
"""Check if a post is bookmarked by the current user."""
try:
bookmark_row = Bookmark.query.filter_by(
user_id=current_user.id, post_uuid=post_uuid
).first()
return jsonify({"bookmarked": bookmark_row is not None})
except Exception as e:
logger.error(f"Error checking bookmark status: {e}")
return jsonify({"error": "Failed to check bookmark status"}), 500
@bp.get("/filters")
def filters():
"""Get available filtersets."""
try:
filter_rows = []
current_filter = "no_filter"
if current_user.is_authenticated:
try:
user_settings = json.loads(current_user.settings) if current_user.settings else {}
current_filter = user_settings.get("filter_set", "no_filter")
except (json.JSONDecodeError, TypeError):
pass
filter_engine = get_filter_engine()
for filterset_name in filter_engine.get_available_filtersets():
filterset_config = filter_engine.config.get_filterset(filterset_name)
if filterset_config:
filter_rows.append(
{
"id": filterset_name,
"name": FILTER_NAME_MAP.get(
filterset_name, filterset_name.replace("_", " ").title()
),
"description": filterset_config.get("description", ""),
"icon": FILTER_ICON_MAP.get(filterset_name, "\U0001f527"),
"active": filterset_name == current_filter,
}
)
return jsonify({"filters": filter_rows})
except Exception as e:
logger.error(f"Error getting filters: {e}")
return jsonify({"error": "Failed to get filters"}), 500
return bp