diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 12c1276..ef7af18 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -5,7 +5,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.dependencies import get_current_admin from app.database import get_db from app.models import Bill, BillBrief, BillDocument, Follow +from app.models.member import Member from app.models.user import User +from app.models.vote import BillVote from app.schemas.schemas import UserResponse router = APIRouter() @@ -134,6 +136,30 @@ async def get_stats( bills_missing_actions = (await db.execute( text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL") )).scalar() + # Member sync health + total_members = (await db.execute(select(func.count()).select_from(Member))).scalar() + house_count = (await db.execute( + select(func.count()).select_from(Member).where(Member.chamber == "House of Representatives") + )).scalar() + senate_count = (await db.execute( + select(func.count()).select_from(Member).where(Member.chamber == "Senate") + )).scalar() + members_missing_chamber = (await db.execute( + select(func.count()).select_from(Member).where(Member.chamber.is_(None)) + )).scalar() + # Vote / alignment coverage + total_votes = (await db.execute(select(func.count()).select_from(BillVote))).scalar() + stanced_row = (await db.execute(text(""" + SELECT + COUNT(DISTINCT f.follow_value) AS total, + COUNT(DISTINCT bv.bill_id) AS with_votes + FROM follows f + LEFT JOIN bill_votes bv ON bv.bill_id = f.follow_value + WHERE f.follow_type = 'bill' + AND f.follow_mode IN ('pocket_veto', 'pocket_boost') + """))).fetchone() + stanced_bills_total = stanced_row.total if stanced_row else 0 + stanced_bills_with_votes = stanced_row.with_votes if stanced_row else 0 # Cited brief points (objects) that have no label yet unlabeled_briefs = (await db.execute( text(""" @@ -165,6 +191,13 @@ async def get_stats( "bills_missing_actions": bills_missing_actions, "unlabeled_briefs": unlabeled_briefs, "remaining": total_bills - total_briefs, + "total_members": total_members, + "house_count": house_count, + "senate_count": senate_count, + "members_missing_chamber": members_missing_chamber, + "total_votes": total_votes, + "stanced_bills_total": stanced_bills_total, + "stanced_bills_with_votes": stanced_bills_with_votes, } @@ -277,6 +310,27 @@ async def trigger_trend_scores(current_user: User = Depends(get_current_admin)): return {"task_id": task.id, "status": "queued"} +@router.post("/trigger-fetch-news") +async def trigger_fetch_news(current_user: User = Depends(get_current_admin)): + from app.workers.news_fetcher import fetch_news_for_active_bills + task = fetch_news_for_active_bills.delay() + return {"task_id": task.id, "status": "queued"} + + +@router.post("/trigger-fetch-votes") +async def trigger_fetch_votes(current_user: User = Depends(get_current_admin)): + from app.workers.vote_fetcher import fetch_votes_for_stanced_bills + task = fetch_votes_for_stanced_bills.delay() + return {"task_id": task.id, "status": "queued"} + + +@router.post("/trigger-member-trend-scores") +async def trigger_member_trend_scores(current_user: User = Depends(get_current_admin)): + from app.workers.member_interest import calculate_all_member_trend_scores + task = calculate_all_member_trend_scores.delay() + return {"task_id": task.id, "status": "queued"} + + @router.post("/bills/{bill_id}/reprocess") async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_admin)): """Queue document and action fetches for a specific bill. Useful for debugging.""" @@ -340,9 +394,12 @@ async def api_health(current_user: User = Depends(get_current_admin)): asyncio.to_thread(_test_newsapi), asyncio.to_thread(_test_gnews), asyncio.to_thread(_test_rep_lookup), + asyncio.to_thread(_test_redis), + asyncio.to_thread(_test_smtp), + asyncio.to_thread(_test_pytrends), return_exceptions=True, ) - keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup"] + keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup", "redis", "smtp", "pytrends"] return { k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)} for k, r in zip(keys, results) @@ -495,6 +552,62 @@ def _test_rep_lookup() -> dict: return {"status": "error", "detail": str(exc)} +def _test_redis() -> dict: + from app.config import settings + import redis as redis_lib + def _call(): + client = redis_lib.from_url(settings.REDIS_URL, socket_timeout=5) + client.ping() + info = client.info("server") + version = info.get("redis_version", "?") + used_mb = round(client.info("memory").get("used_memory", 0) / 1024 / 1024, 1) + return {"status": "ok", "detail": f"Redis {version} โ€” {used_mb} MB used"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + +def _test_smtp() -> dict: + from app.config import settings + if not settings.SMTP_HOST: + return {"status": "skipped", "detail": "SMTP_HOST not configured"} + def _call(): + import smtplib, ssl + if settings.SMTP_STARTTLS: + with smtplib.SMTP(settings.SMTP_HOST, settings.SMTP_PORT, timeout=10) as s: + s.ehlo() + s.starttls(context=ssl.create_default_context()) + s.ehlo() + else: + with smtplib.SMTP_SSL(settings.SMTP_HOST, settings.SMTP_PORT, + context=ssl.create_default_context(), timeout=10) as s: + s.ehlo() + return {"status": "ok", "detail": f"SMTP connected to {settings.SMTP_HOST}:{settings.SMTP_PORT}"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + +def _test_pytrends() -> dict: + from app.config import settings + if not settings.PYTRENDS_ENABLED: + return {"status": "skipped", "detail": "PYTRENDS_ENABLED is false"} + def _call(): + from pytrends.request import TrendReq + pt = TrendReq(hl="en-US", tz=360, timeout=(5, 15)) + pt.build_payload(["congress"], timeframe="now 1-d", geo="US") + data = pt.interest_over_time() + if data is None or data.empty: + return {"status": "error", "detail": "pytrends returned empty data โ€” may be rate-limited"} + return {"status": "ok", "detail": f"pytrends reachable โ€” {len(data)} data points returned"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + @router.get("/task-status/{task_id}") async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)): from app.workers.celery_app import celery_app diff --git a/frontend/app/settings/page.tsx b/frontend/app/settings/page.tsx index 37ea28b..301ec11 100644 --- a/frontend/app/settings/page.tsx +++ b/frontend/app/settings/page.tsx @@ -380,6 +380,9 @@ export default function SettingsPage() { { label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "๐Ÿ”„", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined }, { label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "โš ๏ธ", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined }, { label: "Briefs with unlabeled points", value: stats.unlabeled_briefs, color: stats.unlabeled_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "๐Ÿท๏ธ", action: stats.unlabeled_briefs > 0 ? "Backfill Labels" : undefined }, + { label: "Members synced", value: stats.total_members, color: "text-foreground", icon: "๐Ÿ›๏ธ", note: `${stats.house_count} House ยท ${stats.senate_count} Senate${stats.members_missing_chamber > 0 ? ` ยท โš ๏ธ ${stats.members_missing_chamber} missing chamber` : ""}` }, + { label: "Roll-call votes stored", value: stats.total_votes, color: "text-foreground", icon: "๐Ÿ—ณ๏ธ" }, + { label: "Stanced bills with votes", value: stats.stanced_bills_with_votes, color: "text-foreground", icon: "โš–๏ธ", note: stats.stanced_bills_total > 0 ? `${stats.stanced_bills_with_votes} of ${stats.stanced_bills_total} stanced bills have vote data` : "No stanced bills yet", action: stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "Fetch Votes" : undefined }, ].map(({ label, value, color, icon, note, action }) => (
@@ -688,6 +691,9 @@ export default function SettingsPage() { { key: "newsapi", label: "NewsAPI.org" }, { key: "google_news", label: "Google News RSS" }, { key: "rep_lookup", label: "Rep Lookup (Nominatim + TIGERweb)" }, + { key: "redis", label: "Redis" }, + { key: "smtp", label: "SMTP (Email)" }, + { key: "pytrends", label: "Google Trends (pytrends)" }, ].map(({ key, label }) => { const r = healthData[key]; if (!r) return null; @@ -826,6 +832,29 @@ export default function SettingsPage() { fn: adminAPI.triggerTrendScores, status: "on-demand", }, + { + key: "fetch-news", + name: "Fetch News", + description: "Pull recent news articles for all active bills from NewsAPI and Google News. Runs automatically every 6 hours.", + fn: adminAPI.triggerFetchNews, + status: "on-demand", + }, + { + key: "fetch-votes", + name: "Fetch Votes", + description: "Fetch roll-call votes for all bills you have stanced (Pocket Veto or Pocket Boost). Required for Alignment to show data. Re-fetches bills with new activity since the last run.", + fn: adminAPI.triggerFetchVotes, + status: stats ? (stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "needed" : "on-demand") : "on-demand", + count: stats ? stats.stanced_bills_total - stats.stanced_bills_with_votes : undefined, + countLabel: "stanced bills missing votes", + }, + { + key: "member-trends", + name: "Member Trend Scores", + description: "Calculate newsworthiness scores for legislators based on recent news coverage. Runs automatically nightly.", + fn: adminAPI.triggerMemberTrendScores, + status: "on-demand", + }, { key: "actions", name: "Fetch Bill Actions", diff --git a/frontend/lib/api.ts b/frontend/lib/api.ts index 117fe7e..d32cd02 100644 --- a/frontend/lib/api.ts +++ b/frontend/lib/api.ts @@ -208,6 +208,10 @@ export interface ApiHealth { govinfo: ApiHealthResult; newsapi: ApiHealthResult; google_news: ApiHealthResult; + rep_lookup: ApiHealthResult; + redis: ApiHealthResult; + smtp: ApiHealthResult; + pytrends: ApiHealthResult; } export interface AnalysisStats { @@ -224,6 +228,13 @@ export interface AnalysisStats { bills_missing_actions: number; unlabeled_briefs: number; remaining: number; + total_members: number; + house_count: number; + senate_count: number; + members_missing_chamber: number; + total_votes: number; + stanced_bills_total: number; + stanced_bills_with_votes: number; } export interface NotificationTestResult { @@ -277,6 +288,12 @@ export const adminAPI = { apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data), triggerTrendScores: () => apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data), + triggerFetchNews: () => + apiClient.post("/api/admin/trigger-fetch-news").then((r) => r.data), + triggerFetchVotes: () => + apiClient.post("/api/admin/trigger-fetch-votes").then((r) => r.data), + triggerMemberTrendScores: () => + apiClient.post("/api/admin/trigger-member-trend-scores").then((r) => r.data), backfillSponsors: () => apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data), backfillCitations: () =>