Implements the parallel.md workstreams (Agents A-E) toward REFACTOR_GOAL.md. Phase 1 — app factory + blueprints + /api/v1: - app.py -> create_app() factory (no module-level app); entrypoints updated - routes/ (auth, pages, settings, admin, assets) + blueprints/api.py at /api/v1 - config.py / extensions.py / security.py extracted; services/ layer added - endpoint names preserved so template url_for() calls keep resolving (static check: all 27 template url_for endpoints are defined routes) Phase 2 — one pluggable filter system: - filter_pipeline/registry.py: @register_stage / @register_plugin + discover_modules - engine._init_stages() instantiates registered stages (no hardcoded dict); process_batch is AI-aware: only short-circuits to the AI-disabled path when a filterset's stages declare requires_ai, so offline filtersets run with AI off - BaseFilterPlugin gets a consumer (stages/plugins.py); Keyword/Quality re-enabled via filter_config.json plugins config - comment tree modes ported to stages/comment_filter.py + shared rules.py; wired into /api/v1/posts/<uuid> and /api/v1/comments/<uuid> via FilterEngine.filter_comments() (fails open) - offline quality_filter filterset exercises plugins+ranker without AI - legacy filter_lib / comment_lib / html_generation_lib / generate_html / active_html path deleted Phase 3 prep — pluggable fetchers + Postgres models: - Post / Comment SQLAlchemy models added to models.py - migrate_content_to_db.py backfill (idempotent by uuid, batched, --dry-run) - platforms/ fetcher registry (extension point) - live reads/writes still go through PostService (disk JSON); cutover deferred Phase 6 — test harness: - pytest.ini + tests/ (conftest with in-memory SQLite fixture, no Postgres; stubbed polling/filter singletons) - test_app_factory.py (route registration, no module-level app), test_api_contracts.py (posts/post_detail/comments/filters shape with monkeypatched post_service + get_filter_engine), test_filter_pipeline.py + test_plugin_contract.py (Flask-free; validated locally 12/12 incl. drop-in stage/plugin discovered with zero core edits) Other: .gitignore added (__pycache__, data/, secrets); pytest in requirements. Verification: py_compile clean across the project; the Flask-free filter-pipeline and plugin-contract tests pass locally. App-factory / API-contract tests need deps+docker to run; runtime flask routes / Auth0-repeated-create_app also gated on docker. Co-Authored-By: Claude <noreply@anthropic.com>
464 lines
20 KiB
Python
464 lines
20 KiB
Python
"""Versioned JSON API blueprint."""
|
|
|
|
import json
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
|
|
from flask import Blueprint, jsonify, request
|
|
from flask_login import current_user, login_required
|
|
|
|
from config import DEFAULT_PAGE_SIZE
|
|
from database import db
|
|
from extensions import get_filter_engine
|
|
from models import Bookmark
|
|
from security import is_safe_filterset
|
|
from services import get_display_name_for_source, load_platform_config, post_service
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
FILTER_ICON_MAP = {
|
|
"no_filter": "\U0001f310",
|
|
"safe_content": "\u2705",
|
|
"tech_only": "\U0001f4bb",
|
|
"high_quality": "\u2b50",
|
|
"custom_example": "\U0001f3af",
|
|
}
|
|
|
|
|
|
def _current_filterset():
|
|
"""Return selected filterset from query override or current user settings."""
|
|
filter_override = request.args.get("filter", "")
|
|
if filter_override and is_safe_filterset(filter_override):
|
|
return filter_override
|
|
|
|
if current_user.is_authenticated:
|
|
try:
|
|
user_settings = json.loads(current_user.settings) if current_user.settings else {}
|
|
return user_settings.get("filter_set", "no_filter")
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
return "no_filter"
|
|
|
|
FILTER_NAME_MAP = {
|
|
"no_filter": "All Content",
|
|
"safe_content": "Safe Content",
|
|
"tech_only": "Tech Only",
|
|
"high_quality": "High Quality",
|
|
"custom_example": "Custom Example",
|
|
}
|
|
|
|
|
|
def create_api_blueprint(name="api"):
|
|
"""Create the API blueprint so it can be mounted with a version prefix."""
|
|
bp = Blueprint(name, __name__)
|
|
|
|
@bp.get("/posts")
|
|
def posts():
|
|
"""Get paginated posts with filtering."""
|
|
try:
|
|
platform_config = load_platform_config()
|
|
page = int(request.args.get("page", 1))
|
|
per_page = int(request.args.get("per_page", DEFAULT_PAGE_SIZE))
|
|
community = request.args.get("community", "")
|
|
platform = request.args.get("platform", "")
|
|
search_query = request.args.get("q", "").lower().strip()
|
|
filter_override = request.args.get("filter", "")
|
|
|
|
filterset_name = "no_filter"
|
|
user_communities = []
|
|
time_filter_enabled = False
|
|
time_filter_days = 7
|
|
if current_user.is_authenticated:
|
|
try:
|
|
user_settings = json.loads(current_user.settings) if current_user.settings else {}
|
|
filterset_name = user_settings.get("filter_set", "no_filter")
|
|
user_communities = user_settings.get("communities", [])
|
|
experience_settings = user_settings.get("experience", {})
|
|
time_filter_enabled = experience_settings.get("time_filter_enabled", False)
|
|
time_filter_days = experience_settings.get("time_filter_days", 7)
|
|
except (json.JSONDecodeError, TypeError):
|
|
filterset_name = "no_filter"
|
|
user_communities = []
|
|
time_filter_enabled = False
|
|
time_filter_days = 7
|
|
|
|
if filter_override and is_safe_filterset(filter_override):
|
|
filterset_name = filter_override
|
|
|
|
cached_posts, cached_comments = post_service.load()
|
|
|
|
time_cutoff = None
|
|
if time_filter_enabled:
|
|
cutoff_date = datetime.utcnow() - timedelta(days=time_filter_days)
|
|
time_cutoff = cutoff_date.timestamp()
|
|
|
|
raw_posts = []
|
|
for post_data in cached_posts.values():
|
|
if time_filter_enabled and time_cutoff:
|
|
post_timestamp = post_data.get("timestamp", 0)
|
|
if post_timestamp < time_cutoff:
|
|
continue
|
|
|
|
if community and post_data.get("source", "").lower() != community.lower():
|
|
continue
|
|
|
|
if platform and post_data.get("platform", "").lower() != platform.lower():
|
|
continue
|
|
|
|
if user_communities:
|
|
post_source = post_data.get("source", "").lower()
|
|
post_platform = post_data.get("platform", "").lower()
|
|
post_id = post_data.get("id", "").lower()
|
|
matches_community = any(
|
|
post_source == selected.lower()
|
|
or post_platform == selected.lower()
|
|
or selected.lower() in post_source
|
|
or selected.lower() in post_id
|
|
for selected in user_communities
|
|
if isinstance(selected, str)
|
|
)
|
|
if not matches_community:
|
|
continue
|
|
|
|
if search_query:
|
|
title = post_data.get("title", "").lower()
|
|
content = post_data.get("content", "").lower()
|
|
author = post_data.get("author", "").lower()
|
|
source = post_data.get("source", "").lower()
|
|
if not (
|
|
search_query in title
|
|
or search_query in content
|
|
or search_query in author
|
|
or search_query in source
|
|
):
|
|
continue
|
|
|
|
raw_posts.append(post_data)
|
|
|
|
filtered_posts = get_filter_engine().apply_filterset(
|
|
raw_posts, filterset_name, use_cache=True
|
|
)
|
|
|
|
response_posts = []
|
|
for post_data in filtered_posts:
|
|
post_uuid = post_data.get("uuid")
|
|
source_display = get_display_name_for_source(
|
|
post_data.get("platform", ""),
|
|
post_data.get("source", ""),
|
|
platform_config,
|
|
)
|
|
response_posts.append(
|
|
{
|
|
"id": post_uuid,
|
|
"title": post_data.get("title", "Untitled"),
|
|
"author": post_data.get("author", "Unknown"),
|
|
"platform": post_data.get("platform", "unknown"),
|
|
"score": post_data.get("score", 0),
|
|
"timestamp": post_data.get("timestamp", 0),
|
|
"url": f"/post/{post_uuid}",
|
|
"comments_count": len(cached_comments.get(post_uuid, [])),
|
|
"content_preview": (post_data.get("content", "") or "")[:200] + "..." if post_data.get("content") else "",
|
|
"source": post_data.get("source", ""),
|
|
"source_display": source_display,
|
|
"tags": post_data.get("tags", []),
|
|
"external_url": post_data.get("url", ""),
|
|
"filter_score": post_data.get("_filter_score", 0.5),
|
|
"filter_categories": post_data.get("_filter_categories", []),
|
|
"filter_tags": post_data.get("_filter_tags", []),
|
|
}
|
|
)
|
|
|
|
response_posts.sort(key=lambda x: (x["filter_score"], x["timestamp"]), reverse=True)
|
|
total_posts = len(response_posts)
|
|
start_idx = (page - 1) * per_page
|
|
end_idx = start_idx + per_page
|
|
total_pages = (total_posts + per_page - 1) // per_page
|
|
|
|
return jsonify(
|
|
{
|
|
"posts": response_posts[start_idx:end_idx],
|
|
"pagination": {
|
|
"current_page": page,
|
|
"total_pages": total_pages,
|
|
"total_posts": total_posts,
|
|
"per_page": per_page,
|
|
"has_next": page < total_pages,
|
|
"has_prev": page > 1,
|
|
},
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Error loading posts: {e}")
|
|
return jsonify(
|
|
{
|
|
"posts": [],
|
|
"error": str(e),
|
|
"pagination": {
|
|
"current_page": 1,
|
|
"total_pages": 0,
|
|
"total_posts": 0,
|
|
"per_page": DEFAULT_PAGE_SIZE,
|
|
"has_next": False,
|
|
"has_prev": False,
|
|
},
|
|
}
|
|
)
|
|
|
|
@bp.get("/posts/<post_uuid>")
|
|
def post_detail(post_uuid):
|
|
"""Get one post with its comment tree."""
|
|
try:
|
|
platform_config = load_platform_config()
|
|
cached_posts, cached_comments = post_service.load()
|
|
post_data = cached_posts.get(post_uuid)
|
|
if not post_data:
|
|
return jsonify({"error": "Post not found"}), 404
|
|
|
|
post = dict(post_data)
|
|
post["source_display"] = get_display_name_for_source(
|
|
post.get("platform", ""), post.get("source", ""), platform_config
|
|
)
|
|
filterset_name = _current_filterset()
|
|
filtered_comments = get_filter_engine().filter_comments(
|
|
cached_comments.get(post_uuid, []), filterset_name
|
|
)
|
|
comments = post_service.build_comment_tree(filtered_comments)
|
|
return jsonify({"post": post, "comments": comments})
|
|
except Exception as e:
|
|
logger.error(f"Error loading post {post_uuid}: {e}")
|
|
return jsonify({"error": "Failed to load post"}), 500
|
|
|
|
@bp.get("/comments/<post_uuid>")
|
|
def comments(post_uuid):
|
|
"""Get comments for a post as a tree."""
|
|
try:
|
|
_, cached_comments = post_service.load()
|
|
filterset_name = _current_filterset()
|
|
filtered_comments = get_filter_engine().filter_comments(
|
|
cached_comments.get(post_uuid, []), filterset_name
|
|
)
|
|
return jsonify({"comments": post_service.build_comment_tree(filtered_comments)})
|
|
except Exception as e:
|
|
logger.error(f"Error loading comments for {post_uuid}: {e}")
|
|
return jsonify({"error": "Failed to load comments"}), 500
|
|
|
|
@bp.get("/platforms")
|
|
def platforms():
|
|
"""Get platform configuration and available communities."""
|
|
try:
|
|
platform_config = load_platform_config()
|
|
communities = []
|
|
for key, count in post_service.source_counts().items():
|
|
platform, source = key.split(":", 1)
|
|
platform_info = platform_config.get("platforms", {}).get(platform, {})
|
|
community_info = None
|
|
if platform_info.get("supports_communities"):
|
|
for community in platform_info.get("communities", []):
|
|
if community["id"] == source:
|
|
community_info = community
|
|
break
|
|
|
|
if community_info:
|
|
communities.append(
|
|
{
|
|
"platform": platform,
|
|
"id": source,
|
|
"name": community_info["name"],
|
|
"display_name": community_info["display_name"],
|
|
"icon": community_info.get("icon", platform_info.get("icon", "\U0001f4c4")),
|
|
"count": count,
|
|
"description": community_info.get("description", ""),
|
|
}
|
|
)
|
|
else:
|
|
display_name = get_display_name_for_source(platform, source, platform_config)
|
|
communities.append(
|
|
{
|
|
"platform": platform,
|
|
"id": source,
|
|
"name": source or platform,
|
|
"display_name": display_name,
|
|
"icon": platform_info.get("icon", "\U0001f4c4"),
|
|
"count": count,
|
|
"description": f"Posts from {display_name}",
|
|
}
|
|
)
|
|
|
|
communities.sort(key=lambda x: x["count"], reverse=True)
|
|
return jsonify(
|
|
{
|
|
"platforms": platform_config.get("platforms", {}),
|
|
"communities": communities,
|
|
"total_communities": len(communities),
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Error loading platform configuration: {e}")
|
|
return jsonify({"platforms": {}, "communities": [], "total_communities": 0, "error": str(e)})
|
|
|
|
@bp.get("/content-timestamp")
|
|
def content_timestamp():
|
|
"""Get the last content update timestamp for auto-refresh."""
|
|
try:
|
|
return jsonify({"timestamp": post_service.latest_content_mtime()})
|
|
except Exception as e:
|
|
logger.error(f"Error getting content timestamp: {e}")
|
|
return jsonify({"error": "Failed to get content timestamp"}), 500
|
|
|
|
@bp.post("/bookmark")
|
|
@login_required
|
|
def bookmark():
|
|
"""Toggle bookmark status for a post."""
|
|
try:
|
|
data = request.get_json()
|
|
if not data or "post_uuid" not in data:
|
|
return jsonify({"error": "Missing post_uuid"}), 400
|
|
|
|
post_uuid = data["post_uuid"]
|
|
if not post_uuid:
|
|
return jsonify({"error": "Invalid post_uuid"}), 400
|
|
|
|
existing_bookmark = Bookmark.query.filter_by(
|
|
user_id=current_user.id, post_uuid=post_uuid
|
|
).first()
|
|
if existing_bookmark:
|
|
db.session.delete(existing_bookmark)
|
|
db.session.commit()
|
|
return jsonify({"bookmarked": False, "message": "Bookmark removed"})
|
|
|
|
cached_posts, _ = post_service.load()
|
|
post_data = cached_posts.get(post_uuid, {})
|
|
new_bookmark = Bookmark(
|
|
user_id=current_user.id,
|
|
post_uuid=post_uuid,
|
|
title=post_data.get("title", ""),
|
|
platform=post_data.get("platform", ""),
|
|
source=post_data.get("source", ""),
|
|
)
|
|
db.session.add(new_bookmark)
|
|
db.session.commit()
|
|
return jsonify({"bookmarked": True, "message": "Bookmark added"})
|
|
except Exception as e:
|
|
db.session.rollback()
|
|
logger.error(f"Error toggling bookmark: {e}")
|
|
return jsonify({"error": "Failed to toggle bookmark"}), 500
|
|
|
|
@bp.get("/bookmarks")
|
|
@login_required
|
|
def bookmarks():
|
|
"""Get the current user's bookmarks."""
|
|
try:
|
|
page = int(request.args.get("page", 1))
|
|
per_page = int(request.args.get("per_page", DEFAULT_PAGE_SIZE))
|
|
bookmarks_query = Bookmark.query.filter_by(user_id=current_user.id).order_by(
|
|
Bookmark.created_at.desc()
|
|
)
|
|
total_bookmarks = bookmarks_query.count()
|
|
bookmark_rows = bookmarks_query.offset((page - 1) * per_page).limit(per_page).all()
|
|
cached_posts, cached_comments = post_service.load()
|
|
|
|
bookmark_posts = []
|
|
for bookmark_row in bookmark_rows:
|
|
post_data = cached_posts.get(bookmark_row.post_uuid)
|
|
if post_data:
|
|
bookmark_posts.append(
|
|
{
|
|
"id": bookmark_row.post_uuid,
|
|
"title": post_data.get("title", bookmark_row.title or "Untitled"),
|
|
"author": post_data.get("author", "Unknown"),
|
|
"platform": post_data.get("platform", bookmark_row.platform or "unknown"),
|
|
"score": post_data.get("score", 0),
|
|
"timestamp": post_data.get("timestamp", 0),
|
|
"url": f"/post/{bookmark_row.post_uuid}",
|
|
"comments_count": len(cached_comments.get(bookmark_row.post_uuid, [])),
|
|
"content_preview": (post_data.get("content", "") or "")[:200] + "..." if post_data.get("content") else "",
|
|
"source": post_data.get("source", bookmark_row.source or ""),
|
|
"bookmarked_at": bookmark_row.created_at.isoformat(),
|
|
"external_url": post_data.get("url", ""),
|
|
}
|
|
)
|
|
else:
|
|
bookmark_posts.append(
|
|
{
|
|
"id": bookmark_row.post_uuid,
|
|
"title": bookmark_row.title or "Untitled",
|
|
"author": "Unknown",
|
|
"platform": bookmark_row.platform or "unknown",
|
|
"score": 0,
|
|
"timestamp": 0,
|
|
"url": f"/post/{bookmark_row.post_uuid}",
|
|
"comments_count": 0,
|
|
"content_preview": "Content no longer available",
|
|
"source": bookmark_row.source or "",
|
|
"bookmarked_at": bookmark_row.created_at.isoformat(),
|
|
"external_url": "",
|
|
"archived": True,
|
|
}
|
|
)
|
|
|
|
total_pages = (total_bookmarks + per_page - 1) // per_page
|
|
return jsonify(
|
|
{
|
|
"posts": bookmark_posts,
|
|
"pagination": {
|
|
"current_page": page,
|
|
"total_pages": total_pages,
|
|
"total_posts": total_bookmarks,
|
|
"per_page": per_page,
|
|
"has_next": page < total_pages,
|
|
"has_prev": page > 1,
|
|
},
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.error(f"Error getting bookmarks: {e}")
|
|
return jsonify({"error": "Failed to get bookmarks"}), 500
|
|
|
|
@bp.get("/bookmark-status/<post_uuid>")
|
|
@login_required
|
|
def bookmark_status(post_uuid):
|
|
"""Check if a post is bookmarked by the current user."""
|
|
try:
|
|
bookmark_row = Bookmark.query.filter_by(
|
|
user_id=current_user.id, post_uuid=post_uuid
|
|
).first()
|
|
return jsonify({"bookmarked": bookmark_row is not None})
|
|
except Exception as e:
|
|
logger.error(f"Error checking bookmark status: {e}")
|
|
return jsonify({"error": "Failed to check bookmark status"}), 500
|
|
|
|
@bp.get("/filters")
|
|
def filters():
|
|
"""Get available filtersets."""
|
|
try:
|
|
filter_rows = []
|
|
current_filter = "no_filter"
|
|
if current_user.is_authenticated:
|
|
try:
|
|
user_settings = json.loads(current_user.settings) if current_user.settings else {}
|
|
current_filter = user_settings.get("filter_set", "no_filter")
|
|
except (json.JSONDecodeError, TypeError):
|
|
pass
|
|
|
|
filter_engine = get_filter_engine()
|
|
for filterset_name in filter_engine.get_available_filtersets():
|
|
filterset_config = filter_engine.config.get_filterset(filterset_name)
|
|
if filterset_config:
|
|
filter_rows.append(
|
|
{
|
|
"id": filterset_name,
|
|
"name": FILTER_NAME_MAP.get(
|
|
filterset_name, filterset_name.replace("_", " ").title()
|
|
),
|
|
"description": filterset_config.get("description", ""),
|
|
"icon": FILTER_ICON_MAP.get(filterset_name, "\U0001f527"),
|
|
"active": filterset_name == current_filter,
|
|
}
|
|
)
|
|
return jsonify({"filters": filter_rows})
|
|
except Exception as e:
|
|
logger.error(f"Error getting filters: {e}")
|
|
return jsonify({"error": "Failed to get filters"}), 500
|
|
|
|
return bp |