From 5eebc2f1963a93ce0c74b410ed833abfdcd5ed72 Mon Sep 17 00:00:00 2001 From: Jack Levy Date: Sun, 1 Mar 2026 11:06:14 -0500 Subject: [PATCH] Add bill action pipeline, admin health panel, and LLM provider fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fetch bill actions from Congress.gov and populate the action timeline - Add nightly batch task and beat schedule for active bill actions - Add admin reprocess endpoint for per-bill debugging - Add BriefPanel with "What Changed" view and version history - Add External API Health section with per-source latency testing - Redesign Manual Controls as health panel with status dots and descriptions - Add Resume Analysis task for stalled LLM jobs - Add Backfill Dates & Links task for bills with null metadata - Fix LLM provider/model DB overrides being ignored (env vars used instead) - Fix Gemini 404: gemini-1.5-pro deprecated → gemini-2.0-flash - Fix Anthropic models list: use REST API directly (SDK too old for .models) - Replace test-LLM full analysis with lightweight ping (max_tokens=20) - Add has_document field to BillDetail; show "No bill text published" state - Fix "Introduced: —" showing for bills with null introduced_date - Add bills_missing_sponsor and bills_missing_metadata to admin stats - Add GovInfo health check using /collections endpoint (fixes 500 from /packages) Authored-By: Jack Levy --- backend/app/api/admin.py | 151 ++++++++++++++ backend/app/api/bills.py | 6 +- backend/app/schemas/schemas.py | 18 ++ backend/app/workers/celery_app.py | 6 + backend/app/workers/congress_poller.py | 80 +++++++- backend/app/workers/llm_processor.py | 52 +++++ frontend/app/bills/[id]/page.tsx | 44 +++- frontend/app/settings/page.tsx | 272 +++++++++++++++++++------ frontend/lib/api.ts | 23 +++ frontend/lib/types.ts | 8 + 10 files changed, 586 insertions(+), 74 deletions(-) diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 12add69..b2e1a17 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -106,6 +106,30 @@ async def get_stats( AND jsonb_typeof(key_points->0) = 'string' """) )).scalar() + # Bills with null sponsor + bills_missing_sponsor = (await db.execute( + text("SELECT COUNT(*) FROM bills WHERE sponsor_id IS NULL") + )).scalar() + # Bills with null metadata (introduced_date / chamber / congress_url) + bills_missing_metadata = (await db.execute( + text("SELECT COUNT(*) FROM bills WHERE introduced_date IS NULL OR chamber IS NULL OR congress_url IS NULL") + )).scalar() + # Bills with no document record at all (text not yet published on GovInfo) + no_text_bills = (await db.execute( + text(""" + SELECT COUNT(*) FROM bills b + LEFT JOIN bill_documents bd ON bd.bill_id = b.bill_id + WHERE bd.id IS NULL + """) + )).scalar() + # Documents that have text but no brief (LLM not yet run / failed) + pending_llm = (await db.execute( + text(""" + SELECT COUNT(*) FROM bill_documents bd + LEFT JOIN bill_briefs bb ON bb.document_id = bd.id + WHERE bb.id IS NULL AND bd.raw_text IS NOT NULL + """) + )).scalar() return { "total_bills": total_bills, "docs_fetched": docs_fetched, @@ -113,6 +137,10 @@ async def get_stats( "full_briefs": full_briefs, "amendment_briefs": amendment_briefs, "uncited_briefs": uncited_briefs, + "no_text_bills": no_text_bills, + "pending_llm": pending_llm, + "bills_missing_sponsor": bills_missing_sponsor, + "bills_missing_metadata": bills_missing_metadata, "remaining": total_bills - total_briefs, } @@ -155,6 +183,22 @@ async def trigger_fetch_actions(current_user: User = Depends(get_current_admin)) return {"task_id": task.id, "status": "queued"} +@router.post("/backfill-metadata") +async def backfill_metadata(current_user: User = Depends(get_current_admin)): + """Fill in null introduced_date, congress_url, chamber for existing bills.""" + from app.workers.congress_poller import backfill_bill_metadata + task = backfill_bill_metadata.delay() + return {"task_id": task.id, "status": "queued"} + + +@router.post("/resume-analysis") +async def resume_analysis(current_user: User = Depends(get_current_admin)): + """Re-queue LLM processing for docs with no brief, and document fetching for bills with no doc.""" + from app.workers.llm_processor import resume_pending_analysis + task = resume_pending_analysis.delay() + return {"task_id": task.id, "status": "queued"} + + @router.post("/trigger-trend-scores") async def trigger_trend_scores(current_user: User = Depends(get_current_admin)): from app.workers.trend_scorer import calculate_all_trend_scores @@ -172,6 +216,113 @@ async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_ return {"task_ids": {"documents": doc_task.id, "actions": actions_task.id}} +@router.get("/api-health") +async def api_health(current_user: User = Depends(get_current_admin)): + """Test each external API and return status + latency for each.""" + import asyncio + results = await asyncio.gather( + asyncio.to_thread(_test_congress), + asyncio.to_thread(_test_govinfo), + asyncio.to_thread(_test_newsapi), + asyncio.to_thread(_test_gnews), + return_exceptions=True, + ) + keys = ["congress_gov", "govinfo", "newsapi", "google_news"] + return { + k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)} + for k, r in zip(keys, results) + } + + +def _timed(fn): + """Run fn(), return its dict merged with latency_ms.""" + import time as _time + t0 = _time.perf_counter() + result = fn() + result["latency_ms"] = round((_time.perf_counter() - t0) * 1000) + return result + + +def _test_congress() -> dict: + from app.config import settings + from app.services import congress_api + if not settings.DATA_GOV_API_KEY: + return {"status": "error", "detail": "DATA_GOV_API_KEY not configured"} + def _call(): + data = congress_api.get_bills(119, limit=1) + count = data.get("pagination", {}).get("count") or len(data.get("bills", [])) + return {"status": "ok", "detail": f"{count:,} bills available in 119th Congress"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + +def _test_govinfo() -> dict: + from app.config import settings + import requests as req + if not settings.DATA_GOV_API_KEY: + return {"status": "error", "detail": "DATA_GOV_API_KEY not configured"} + def _call(): + # /collections lists all available collections — simple health check endpoint + resp = req.get( + "https://api.govinfo.gov/collections", + params={"api_key": settings.DATA_GOV_API_KEY}, + timeout=15, + ) + resp.raise_for_status() + data = resp.json() + collections = data.get("collections", []) + bills_col = next((c for c in collections if c.get("collectionCode") == "BILLS"), None) + if bills_col: + count = bills_col.get("packageCount", "?") + return {"status": "ok", "detail": f"BILLS collection: {count:,} packages" if isinstance(count, int) else "GovInfo reachable, BILLS collection found"} + return {"status": "ok", "detail": f"GovInfo reachable — {len(collections)} collections available"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + +def _test_newsapi() -> dict: + from app.config import settings + import requests as req + if not settings.NEWSAPI_KEY: + return {"status": "skipped", "detail": "NEWSAPI_KEY not configured"} + def _call(): + resp = req.get( + "https://newsapi.org/v2/top-headlines", + params={"country": "us", "pageSize": 1, "apiKey": settings.NEWSAPI_KEY}, + timeout=10, + ) + data = resp.json() + if data.get("status") != "ok": + return {"status": "error", "detail": data.get("message", "Unknown error")} + return {"status": "ok", "detail": f"{data.get('totalResults', 0):,} headlines available"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + +def _test_gnews() -> dict: + import requests as req + def _call(): + resp = req.get( + "https://news.google.com/rss/search", + params={"q": "congress", "hl": "en-US", "gl": "US", "ceid": "US:en"}, + timeout=10, + headers={"User-Agent": "Mozilla/5.0"}, + ) + resp.raise_for_status() + item_count = resp.text.count("") + return {"status": "ok", "detail": f"{item_count} items in test RSS feed"} + try: + return _timed(_call) + except Exception as exc: + return {"status": "error", "detail": str(exc)} + + @router.get("/task-status/{task_id}") async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)): from app.workers.celery_app import celery_app diff --git a/backend/app/api/bills.py b/backend/app/api/bills.py index ff89e60..21015a1 100644 --- a/backend/app/api/bills.py +++ b/backend/app/api/bills.py @@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload from app.database import get_db -from app.models import Bill, BillAction, BillBrief, NewsArticle, TrendScore +from app.models import Bill, BillAction, BillBrief, BillDocument, NewsArticle, TrendScore from app.schemas.schemas import ( BillDetailSchema, BillSchema, @@ -109,6 +109,10 @@ async def get_bill(bill_id: str, db: AsyncSession = Depends(get_db)): detail.latest_brief = bill.briefs[0] if bill.trend_scores: detail.latest_trend = bill.trend_scores[0] + doc_exists = await db.scalar( + select(func.count()).select_from(BillDocument).where(BillDocument.bill_id == bill_id) + ) + detail.has_document = bool(doc_exists) # Trigger a background news refresh if no articles are stored but trend # data shows there are gnews results out there waiting to be fetched. diff --git a/backend/app/schemas/schemas.py b/backend/app/schemas/schemas.py index 28ef84c..0743c9d 100644 --- a/backend/app/schemas/schemas.py +++ b/backend/app/schemas/schemas.py @@ -3,6 +3,23 @@ from typing import Any, Generic, Optional, TypeVar from pydantic import BaseModel + +# ── Notifications ────────────────────────────────────────────────────────────── + +class NotificationSettingsResponse(BaseModel): + ntfy_topic_url: str = "" + ntfy_token: str = "" + ntfy_enabled: bool = False + rss_token: Optional[str] = None + + model_config = {"from_attributes": True} + + +class NotificationSettingsUpdate(BaseModel): + ntfy_topic_url: Optional[str] = None + ntfy_token: Optional[str] = None + ntfy_enabled: Optional[bool] = None + T = TypeVar("T") @@ -144,6 +161,7 @@ class BillDetailSchema(BillSchema): news_articles: list[NewsArticleSchema] = [] trend_scores: list[TrendScoreSchema] = [] briefs: list[BriefSchema] = [] + has_document: bool = False # ── Follow ──────────────────────────────────────────────────────────────────── diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index 8cf8796..62be663 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -15,6 +15,7 @@ celery_app = Celery( "app.workers.news_fetcher", "app.workers.trend_scorer", "app.workers.member_interest", + "app.workers.notification_dispatcher", ], ) @@ -37,6 +38,7 @@ celery_app.conf.update( "app.workers.news_fetcher.*": {"queue": "news"}, "app.workers.trend_scorer.*": {"queue": "news"}, "app.workers.member_interest.*": {"queue": "news"}, + "app.workers.notification_dispatcher.*": {"queue": "polling"}, }, task_queues=[ Queue("polling"), @@ -72,5 +74,9 @@ celery_app.conf.update( "task": "app.workers.congress_poller.fetch_actions_for_active_bills", "schedule": crontab(hour=4, minute=0), # 4 AM UTC, after trend + member scoring }, + "dispatch-notifications": { + "task": "app.workers.notification_dispatcher.dispatch_notifications", + "schedule": crontab(minute="*/5"), # Every 5 minutes + }, }, ) diff --git a/backend/app/workers/congress_poller.py b/backend/app/workers/congress_poller.py index 70bc531..151c199 100644 --- a/backend/app/workers/congress_poller.py +++ b/backend/app/workers/congress_poller.py @@ -300,17 +300,95 @@ def fetch_actions_for_active_bills(self): def _update_bill_if_changed(db, existing: Bill, parsed: dict) -> bool: """Update bill fields if anything has changed. Returns True if updated.""" changed = False + dirty = False + + # Meaningful change fields — trigger document + action fetch when updated track_fields = ["title", "short_title", "latest_action_date", "latest_action_text", "status"] for field in track_fields: new_val = parsed.get(field) if new_val and getattr(existing, field) != new_val: setattr(existing, field, new_val) changed = True + dirty = True + + # Static fields — only fill in if currently null; no change trigger needed + fill_null_fields = ["introduced_date", "congress_url", "chamber"] + for field in fill_null_fields: + new_val = parsed.get(field) + if new_val and getattr(existing, field) is None: + setattr(existing, field, new_val) + dirty = True + if changed: existing.last_checked_at = datetime.now(timezone.utc) + if dirty: db.commit() - # Check for new text versions and sync actions now that the bill has changed + if changed: from app.workers.document_fetcher import fetch_bill_documents fetch_bill_documents.delay(existing.bill_id) fetch_bill_actions.delay(existing.bill_id) return changed + + +@celery_app.task(bind=True, name="app.workers.congress_poller.backfill_bill_metadata") +def backfill_bill_metadata(self): + """ + Find bills with null introduced_date (or other static fields) and + re-fetch their detail from Congress.gov to fill in the missing values. + No document or LLM calls — metadata only. + """ + db = get_sync_db() + try: + from sqlalchemy import text as sa_text + rows = db.execute(sa_text(""" + SELECT bill_id, congress_number, bill_type, bill_number + FROM bills + WHERE introduced_date IS NULL + OR congress_url IS NULL + OR chamber IS NULL + """)).fetchall() + + updated = 0 + skipped = 0 + for row in rows: + try: + detail = congress_api.get_bill_detail( + row.congress_number, row.bill_type, row.bill_number + ) + bill_data = detail.get("bill", {}) + parsed = congress_api.parse_bill_from_api( + { + "type": row.bill_type, + "number": row.bill_number, + "introducedDate": bill_data.get("introducedDate"), + "title": bill_data.get("title"), + "shortTitle": bill_data.get("shortTitle"), + "latestAction": bill_data.get("latestAction") or {}, + }, + row.congress_number, + ) + bill = db.get(Bill, row.bill_id) + if not bill: + skipped += 1 + continue + fill_null_fields = ["introduced_date", "congress_url", "chamber", "title", "short_title"] + dirty = False + for field in fill_null_fields: + new_val = parsed.get(field) + if new_val and getattr(bill, field) is None: + setattr(bill, field, new_val) + dirty = True + if dirty: + db.commit() + updated += 1 + else: + skipped += 1 + time.sleep(0.2) # ~300 req/min — well under the 5k/hr limit + except Exception as exc: + logger.warning(f"backfill_bill_metadata: failed for {row.bill_id}: {exc}") + skipped += 1 + + logger.info(f"backfill_bill_metadata: {updated} updated, {skipped} skipped") + return {"updated": updated, "skipped": skipped} + finally: + db.close() diff --git a/backend/app/workers/llm_processor.py b/backend/app/workers/llm_processor.py index 47cc1d8..cd40074 100644 --- a/backend/app/workers/llm_processor.py +++ b/backend/app/workers/llm_processor.py @@ -199,3 +199,55 @@ def backfill_brief_citations(self): return {"total": total, "queued": queued, "skipped": skipped} finally: db.close() + + +@celery_app.task(bind=True, name="app.workers.llm_processor.resume_pending_analysis") +def resume_pending_analysis(self): + """ + Two-pass backfill for bills missing analysis: + + Pass 1 — Documents with no brief (LLM tasks failed/timed out): + Find BillDocuments that have raw_text but no BillBrief, re-queue LLM. + + Pass 2 — Bills with no document at all: + Find Bills with no BillDocument, re-queue document fetch (which will + then chain into LLM if text is available on GovInfo). + """ + db = get_sync_db() + try: + # Pass 1: docs with raw_text but no brief + docs_no_brief = db.execute(text(""" + SELECT bd.id + FROM bill_documents bd + LEFT JOIN bill_briefs bb ON bb.document_id = bd.id + WHERE bb.id IS NULL AND bd.raw_text IS NOT NULL + """)).fetchall() + + queued_llm = 0 + for row in docs_no_brief: + process_document_with_llm.delay(row.id) + queued_llm += 1 + time.sleep(0.1) + + # Pass 2: bills with no document at all + bills_no_doc = db.execute(text(""" + SELECT b.bill_id + FROM bills b + LEFT JOIN bill_documents bd ON bd.bill_id = b.bill_id + WHERE bd.id IS NULL + """)).fetchall() + + queued_fetch = 0 + from app.workers.document_fetcher import fetch_bill_documents + for row in bills_no_doc: + fetch_bill_documents.delay(row.bill_id) + queued_fetch += 1 + time.sleep(0.1) + + logger.info( + f"resume_pending_analysis: {queued_llm} LLM tasks queued, " + f"{queued_fetch} document fetch tasks queued" + ) + return {"queued_llm": queued_llm, "queued_fetch": queued_fetch} + finally: + db.close() diff --git a/frontend/app/bills/[id]/page.tsx b/frontend/app/bills/[id]/page.tsx index 23b21ce..07a0d53 100644 --- a/frontend/app/bills/[id]/page.tsx +++ b/frontend/app/bills/[id]/page.tsx @@ -2,7 +2,7 @@ import { use, useEffect, useRef } from "react"; import Link from "next/link"; -import { ArrowLeft, ExternalLink, User } from "lucide-react"; +import { ArrowLeft, ExternalLink, FileX, User } from "lucide-react"; import { useBill, useBillNews, useBillTrend } from "@/lib/hooks/useBills"; import { BriefPanel } from "@/components/bills/BriefPanel"; import { ActionTimeline } from "@/components/bills/ActionTimeline"; @@ -82,10 +82,12 @@ export default function BillDetailPage({ params }: { params: Promise<{ id: strin {bill.sponsor.state && {bill.sponsor.state}} )} -

- Introduced: {formatDate(bill.introduced_date)} +

+ {bill.introduced_date && ( + Introduced: {formatDate(bill.introduced_date)} + )} {bill.congress_url && ( - + congress.gov )} @@ -97,7 +99,39 @@ export default function BillDetailPage({ params }: { params: Promise<{ id: strin {/* Content grid */}

- + {bill.briefs.length > 0 ? ( + + ) : bill.has_document ? ( +
+

Analysis pending

+

+ Bill text was retrieved but has not yet been analyzed. Check back shortly. +

+
+ ) : ( +
+
+ + No bill text published +
+

+ As of {new Date().toLocaleDateString("en-US", { month: "long", day: "numeric", year: "numeric" })},{" "} + no official text has been received for{" "} + {billLabel(bill.bill_type, bill.bill_number)}. + Analysis will be generated automatically once text is published on Congress.gov. +

+ {bill.congress_url && ( + + Check status on Congress.gov + + )} +
+ )}
diff --git a/frontend/app/settings/page.tsx b/frontend/app/settings/page.tsx index b3e7474..9d43d82 100644 --- a/frontend/app/settings/page.tsx +++ b/frontend/app/settings/page.tsx @@ -20,7 +20,7 @@ import { Copy, Rss, } from "lucide-react"; -import { settingsAPI, adminAPI, notificationsAPI, type AdminUser, type LLMModel } from "@/lib/api"; +import { settingsAPI, adminAPI, notificationsAPI, type AdminUser, type LLMModel, type ApiHealthResult } from "@/lib/api"; import { useAuthStore } from "@/stores/authStore"; const LLM_PROVIDERS = [ @@ -47,6 +47,18 @@ export default function SettingsPage() { refetchInterval: 30_000, }); + const [healthTesting, setHealthTesting] = useState(false); + const [healthData, setHealthData] = useState | null>(null); + const testApiHealth = async () => { + setHealthTesting(true); + try { + const result = await adminAPI.getApiHealth(); + setHealthData(result as unknown as Record); + } finally { + setHealthTesting(false); + } + }; + const { data: users, isLoading: usersLoading } = useQuery({ queryKey: ["admin-users"], queryFn: () => adminAPI.listUsers(), @@ -167,34 +179,16 @@ export default function SettingsPage() { {/* Analysis Status */}

- Analysis Status + Bill Pipeline refreshes every 30s

{stats ? ( <> -
-
- -
{stats.total_bills.toLocaleString()}
-
Total Bills
-
-
- -
{stats.docs_fetched.toLocaleString()}
-
Docs Fetched
-
-
- -
{stats.briefs_generated.toLocaleString()}
-
Briefs Generated
-
-
- {/* Progress bar */}
- {stats.full_briefs} full · {stats.amendment_briefs} amendments - {pct}% analyzed · {stats.remaining.toLocaleString()} remaining + {stats.briefs_generated.toLocaleString()} analyzed ({stats.full_briefs} full · {stats.amendment_briefs} amendments) + {pct}% of {stats.total_bills.toLocaleString()} bills
- {stats.uncited_briefs > 0 && ( -

- ⚠ {stats.uncited_briefs.toLocaleString()} brief{stats.uncited_briefs !== 1 ? "s" : ""} missing citations — run Backfill Citations to fix -

- )} +
+ + {/* Pipeline breakdown table */} +
+ {[ + { label: "Total bills tracked", value: stats.total_bills, color: "text-foreground", icon: "📋" }, + { label: "Text published on Congress.gov", value: stats.docs_fetched, color: "text-blue-600 dark:text-blue-400", icon: "📄" }, + { label: "No text published yet", value: stats.no_text_bills, color: "text-muted-foreground", icon: "⏳", note: "Normal — bill text appears after committee markup" }, + { label: "AI briefs generated", value: stats.briefs_generated, color: "text-green-600 dark:text-green-400", icon: "✅" }, + { label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined }, + { label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined }, + ].map(({ label, value, color, icon, note, action }) => ( +
+
+ {icon} +
+ {label} + {note &&

{note}

} +
+
+
+ {value.toLocaleString()} + {action && ( + → run {action} + )} +
+
+ ))}
) : ( @@ -560,50 +577,171 @@ export default function SettingsPage() {
+ {/* API Health */} +
+
+

External API Health

+ +
+ + {healthData ? ( +
+ {[ + { key: "congress_gov", label: "Congress.gov API" }, + { key: "govinfo", label: "GovInfo API" }, + { key: "newsapi", label: "NewsAPI.org" }, + { key: "google_news", label: "Google News RSS" }, + ].map(({ key, label }) => { + const r = healthData[key]; + if (!r) return null; + return ( +
+
+
{label}
+
+ {r.detail} +
+
+
+ {r.latency_ms !== undefined && ( + {r.latency_ms}ms + )} + {r.status === "ok" && } + {r.status === "error" && } + {r.status === "skipped" && —} +
+
+ ); + })} +
+ ) : ( +

+ Click Run Tests to check connectivity to each external data source. +

+ )} +
+ {/* Manual Controls */}

Manual Controls

-
- - - - - - +
+ {([ + { + key: "poll", + name: "Trigger Poll", + description: "Check Congress.gov for newly introduced or updated bills. Runs automatically on a schedule — use this to force an immediate sync.", + fn: adminAPI.triggerPoll, + status: "on-demand", + }, + { + key: "members", + name: "Sync Members", + description: "Refresh all member profiles from Congress.gov including biography, current term, leadership roles, and contact information.", + fn: adminAPI.triggerMemberSync, + status: "on-demand", + }, + { + key: "trends", + name: "Calculate Trends", + description: "Score bill and member newsworthiness by counting recent news headlines and Google search interest. Updates the trend charts.", + fn: adminAPI.triggerTrendScores, + status: "on-demand", + }, + { + key: "actions", + name: "Fetch Bill Actions", + description: "Download the full legislative history (votes, referrals, amendments) for recently active bills and populate the timeline view.", + fn: adminAPI.triggerFetchActions, + status: "on-demand", + }, + { + key: "sponsors", + name: "Backfill Sponsors", + description: "Link bill sponsors that weren't captured during the initial import. Safe to re-run — skips bills that already have a sponsor.", + fn: adminAPI.backfillSponsors, + status: stats ? (stats.bills_missing_sponsor > 0 ? "needed" : "ok") : "on-demand", + count: stats?.bills_missing_sponsor, + countLabel: "bills missing sponsor", + }, + { + key: "metadata", + name: "Backfill Dates & Links", + description: "Fill in missing introduced dates, chamber assignments, and congress.gov links by re-fetching bill detail from Congress.gov.", + fn: adminAPI.backfillMetadata, + status: stats ? (stats.bills_missing_metadata > 0 ? "needed" : "ok") : "on-demand", + count: stats?.bills_missing_metadata, + countLabel: "bills missing metadata", + }, + { + key: "citations", + name: "Backfill Citations", + description: "Regenerate AI briefs that were created before inline source citations were added. Deletes the old brief and re-runs LLM analysis using the already-stored bill text — no new Congress.gov calls.", + fn: adminAPI.backfillCitations, + status: stats ? (stats.uncited_briefs > 0 ? "needed" : "ok") : "on-demand", + count: stats?.uncited_briefs, + countLabel: "briefs need regeneration", + }, + { + key: "resume", + name: "Resume Analysis", + description: "Restart AI brief generation for bills where processing stalled or failed (e.g. after an LLM quota outage). Also re-queues document fetching for bills that have no text yet.", + fn: adminAPI.resumeAnalysis, + status: stats ? (stats.pending_llm > 0 ? "needed" : "ok") : "on-demand", + count: stats?.pending_llm, + countLabel: "bills pending analysis", + }, + ] as Array<{ + key: string; + name: string; + description: string; + fn: () => Promise<{ task_id: string }>; + status: "ok" | "needed" | "on-demand"; + count?: number; + countLabel?: string; + }>).map(({ key, name, description, fn, status, count, countLabel }) => ( +
+
+
+
+ {name} + {status === "ok" && ( + ✓ Up to date + )} + {status === "needed" && count !== undefined && count > 0 && ( + + ⚠ {count.toLocaleString()} {countLabel} + + )} + {taskIds[key] && ( + queued ✓ + )} +
+

{description}

+
+ +
+ ))}
- {Object.entries(taskIds).map(([name, id]) => ( -

{name}: task {id} queued

- ))}
); diff --git a/frontend/lib/api.ts b/frontend/lib/api.ts index 17d2096..364e3a0 100644 --- a/frontend/lib/api.ts +++ b/frontend/lib/api.ts @@ -137,6 +137,19 @@ export interface AdminUser { created_at: string; } +export interface ApiHealthResult { + status: "ok" | "error" | "skipped"; + detail: string; + latency_ms?: number; +} + +export interface ApiHealth { + congress_gov: ApiHealthResult; + govinfo: ApiHealthResult; + newsapi: ApiHealthResult; + google_news: ApiHealthResult; +} + export interface AnalysisStats { total_bills: number; docs_fetched: number; @@ -144,6 +157,10 @@ export interface AnalysisStats { full_briefs: number; amendment_briefs: number; uncited_briefs: number; + no_text_bills: number; + pending_llm: number; + bills_missing_sponsor: number; + bills_missing_metadata: number; remaining: number; } @@ -182,6 +199,12 @@ export const adminAPI = { apiClient.post("/api/admin/backfill-citations").then((r) => r.data), triggerFetchActions: () => apiClient.post("/api/admin/trigger-fetch-actions").then((r) => r.data), + backfillMetadata: () => + apiClient.post("/api/admin/backfill-metadata").then((r) => r.data), + resumeAnalysis: () => + apiClient.post("/api/admin/resume-analysis").then((r) => r.data), + getApiHealth: () => + apiClient.get("/api/admin/api-health").then((r) => r.data), getTaskStatus: (taskId: string) => apiClient.get(`/api/admin/task-status/${taskId}`).then((r) => r.data), }; diff --git a/frontend/lib/types.ts b/frontend/lib/types.ts index 87a7f44..840d0ad 100644 --- a/frontend/lib/types.ts +++ b/frontend/lib/types.ts @@ -123,6 +123,7 @@ export interface BillDetail extends Bill { news_articles: NewsArticle[]; trend_scores: TrendScore[]; briefs: BriefSchema[]; + has_document: boolean; } export interface PaginatedResponse { @@ -153,3 +154,10 @@ export interface SettingsData { newsapi_enabled: boolean; pytrends_enabled: boolean; } + +export interface NotificationSettings { + ntfy_topic_url: string; + ntfy_token: string; + ntfy_enabled: boolean; + rss_token: string | null; +}