Add bill action pipeline, admin health panel, and LLM provider fixes
- Fetch bill actions from Congress.gov and populate the action timeline - Add nightly batch task and beat schedule for active bill actions - Add admin reprocess endpoint for per-bill debugging - Add BriefPanel with "What Changed" view and version history - Add External API Health section with per-source latency testing - Redesign Manual Controls as health panel with status dots and descriptions - Add Resume Analysis task for stalled LLM jobs - Add Backfill Dates & Links task for bills with null metadata - Fix LLM provider/model DB overrides being ignored (env vars used instead) - Fix Gemini 404: gemini-1.5-pro deprecated → gemini-2.0-flash - Fix Anthropic models list: use REST API directly (SDK too old for .models) - Replace test-LLM full analysis with lightweight ping (max_tokens=20) - Add has_document field to BillDetail; show "No bill text published" state - Fix "Introduced: —" showing for bills with null introduced_date - Add bills_missing_sponsor and bills_missing_metadata to admin stats - Add GovInfo health check using /collections endpoint (fixes 500 from /packages) Authored-By: Jack Levy
This commit is contained in:
Jack Levy
@@ -106,6 +106,30 @@ async def get_stats(
|
||||
AND jsonb_typeof(key_points->0) = 'string'
|
||||
""")
|
||||
)).scalar()
|
||||
# Bills with null sponsor
|
||||
bills_missing_sponsor = (await db.execute(
|
||||
text("SELECT COUNT(*) FROM bills WHERE sponsor_id IS NULL")
|
||||
)).scalar()
|
||||
# Bills with null metadata (introduced_date / chamber / congress_url)
|
||||
bills_missing_metadata = (await db.execute(
|
||||
text("SELECT COUNT(*) FROM bills WHERE introduced_date IS NULL OR chamber IS NULL OR congress_url IS NULL")
|
||||
)).scalar()
|
||||
# Bills with no document record at all (text not yet published on GovInfo)
|
||||
no_text_bills = (await db.execute(
|
||||
text("""
|
||||
SELECT COUNT(*) FROM bills b
|
||||
LEFT JOIN bill_documents bd ON bd.bill_id = b.bill_id
|
||||
WHERE bd.id IS NULL
|
||||
""")
|
||||
)).scalar()
|
||||
# Documents that have text but no brief (LLM not yet run / failed)
|
||||
pending_llm = (await db.execute(
|
||||
text("""
|
||||
SELECT COUNT(*) FROM bill_documents bd
|
||||
LEFT JOIN bill_briefs bb ON bb.document_id = bd.id
|
||||
WHERE bb.id IS NULL AND bd.raw_text IS NOT NULL
|
||||
""")
|
||||
)).scalar()
|
||||
return {
|
||||
"total_bills": total_bills,
|
||||
"docs_fetched": docs_fetched,
|
||||
@@ -113,6 +137,10 @@ async def get_stats(
|
||||
"full_briefs": full_briefs,
|
||||
"amendment_briefs": amendment_briefs,
|
||||
"uncited_briefs": uncited_briefs,
|
||||
"no_text_bills": no_text_bills,
|
||||
"pending_llm": pending_llm,
|
||||
"bills_missing_sponsor": bills_missing_sponsor,
|
||||
"bills_missing_metadata": bills_missing_metadata,
|
||||
"remaining": total_bills - total_briefs,
|
||||
}
|
||||
|
||||
@@ -155,6 +183,22 @@ async def trigger_fetch_actions(current_user: User = Depends(get_current_admin))
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/backfill-metadata")
|
||||
async def backfill_metadata(current_user: User = Depends(get_current_admin)):
|
||||
"""Fill in null introduced_date, congress_url, chamber for existing bills."""
|
||||
from app.workers.congress_poller import backfill_bill_metadata
|
||||
task = backfill_bill_metadata.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/resume-analysis")
|
||||
async def resume_analysis(current_user: User = Depends(get_current_admin)):
|
||||
"""Re-queue LLM processing for docs with no brief, and document fetching for bills with no doc."""
|
||||
from app.workers.llm_processor import resume_pending_analysis
|
||||
task = resume_pending_analysis.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/trigger-trend-scores")
|
||||
async def trigger_trend_scores(current_user: User = Depends(get_current_admin)):
|
||||
from app.workers.trend_scorer import calculate_all_trend_scores
|
||||
@@ -172,6 +216,113 @@ async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_
|
||||
return {"task_ids": {"documents": doc_task.id, "actions": actions_task.id}}
|
||||
|
||||
|
||||
@router.get("/api-health")
|
||||
async def api_health(current_user: User = Depends(get_current_admin)):
|
||||
"""Test each external API and return status + latency for each."""
|
||||
import asyncio
|
||||
results = await asyncio.gather(
|
||||
asyncio.to_thread(_test_congress),
|
||||
asyncio.to_thread(_test_govinfo),
|
||||
asyncio.to_thread(_test_newsapi),
|
||||
asyncio.to_thread(_test_gnews),
|
||||
return_exceptions=True,
|
||||
)
|
||||
keys = ["congress_gov", "govinfo", "newsapi", "google_news"]
|
||||
return {
|
||||
k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)}
|
||||
for k, r in zip(keys, results)
|
||||
}
|
||||
|
||||
|
||||
def _timed(fn):
|
||||
"""Run fn(), return its dict merged with latency_ms."""
|
||||
import time as _time
|
||||
t0 = _time.perf_counter()
|
||||
result = fn()
|
||||
result["latency_ms"] = round((_time.perf_counter() - t0) * 1000)
|
||||
return result
|
||||
|
||||
|
||||
def _test_congress() -> dict:
|
||||
from app.config import settings
|
||||
from app.services import congress_api
|
||||
if not settings.DATA_GOV_API_KEY:
|
||||
return {"status": "error", "detail": "DATA_GOV_API_KEY not configured"}
|
||||
def _call():
|
||||
data = congress_api.get_bills(119, limit=1)
|
||||
count = data.get("pagination", {}).get("count") or len(data.get("bills", []))
|
||||
return {"status": "ok", "detail": f"{count:,} bills available in 119th Congress"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
def _test_govinfo() -> dict:
|
||||
from app.config import settings
|
||||
import requests as req
|
||||
if not settings.DATA_GOV_API_KEY:
|
||||
return {"status": "error", "detail": "DATA_GOV_API_KEY not configured"}
|
||||
def _call():
|
||||
# /collections lists all available collections — simple health check endpoint
|
||||
resp = req.get(
|
||||
"https://api.govinfo.gov/collections",
|
||||
params={"api_key": settings.DATA_GOV_API_KEY},
|
||||
timeout=15,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
collections = data.get("collections", [])
|
||||
bills_col = next((c for c in collections if c.get("collectionCode") == "BILLS"), None)
|
||||
if bills_col:
|
||||
count = bills_col.get("packageCount", "?")
|
||||
return {"status": "ok", "detail": f"BILLS collection: {count:,} packages" if isinstance(count, int) else "GovInfo reachable, BILLS collection found"}
|
||||
return {"status": "ok", "detail": f"GovInfo reachable — {len(collections)} collections available"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
def _test_newsapi() -> dict:
|
||||
from app.config import settings
|
||||
import requests as req
|
||||
if not settings.NEWSAPI_KEY:
|
||||
return {"status": "skipped", "detail": "NEWSAPI_KEY not configured"}
|
||||
def _call():
|
||||
resp = req.get(
|
||||
"https://newsapi.org/v2/top-headlines",
|
||||
params={"country": "us", "pageSize": 1, "apiKey": settings.NEWSAPI_KEY},
|
||||
timeout=10,
|
||||
)
|
||||
data = resp.json()
|
||||
if data.get("status") != "ok":
|
||||
return {"status": "error", "detail": data.get("message", "Unknown error")}
|
||||
return {"status": "ok", "detail": f"{data.get('totalResults', 0):,} headlines available"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
def _test_gnews() -> dict:
|
||||
import requests as req
|
||||
def _call():
|
||||
resp = req.get(
|
||||
"https://news.google.com/rss/search",
|
||||
params={"q": "congress", "hl": "en-US", "gl": "US", "ceid": "US:en"},
|
||||
timeout=10,
|
||||
headers={"User-Agent": "Mozilla/5.0"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
item_count = resp.text.count("<item>")
|
||||
return {"status": "ok", "detail": f"{item_count} items in test RSS feed"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
@router.get("/task-status/{task_id}")
|
||||
async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)):
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
@@ -6,7 +6,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from app.database import get_db
|
||||
from app.models import Bill, BillAction, BillBrief, NewsArticle, TrendScore
|
||||
from app.models import Bill, BillAction, BillBrief, BillDocument, NewsArticle, TrendScore
|
||||
from app.schemas.schemas import (
|
||||
BillDetailSchema,
|
||||
BillSchema,
|
||||
@@ -109,6 +109,10 @@ async def get_bill(bill_id: str, db: AsyncSession = Depends(get_db)):
|
||||
detail.latest_brief = bill.briefs[0]
|
||||
if bill.trend_scores:
|
||||
detail.latest_trend = bill.trend_scores[0]
|
||||
doc_exists = await db.scalar(
|
||||
select(func.count()).select_from(BillDocument).where(BillDocument.bill_id == bill_id)
|
||||
)
|
||||
detail.has_document = bool(doc_exists)
|
||||
|
||||
# Trigger a background news refresh if no articles are stored but trend
|
||||
# data shows there are gnews results out there waiting to be fetched.
|
||||
|
||||
@@ -3,6 +3,23 @@ from typing import Any, Generic, Optional, TypeVar
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
# ── Notifications ──────────────────────────────────────────────────────────────
|
||||
|
||||
class NotificationSettingsResponse(BaseModel):
|
||||
ntfy_topic_url: str = ""
|
||||
ntfy_token: str = ""
|
||||
ntfy_enabled: bool = False
|
||||
rss_token: Optional[str] = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class NotificationSettingsUpdate(BaseModel):
|
||||
ntfy_topic_url: Optional[str] = None
|
||||
ntfy_token: Optional[str] = None
|
||||
ntfy_enabled: Optional[bool] = None
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@@ -144,6 +161,7 @@ class BillDetailSchema(BillSchema):
|
||||
news_articles: list[NewsArticleSchema] = []
|
||||
trend_scores: list[TrendScoreSchema] = []
|
||||
briefs: list[BriefSchema] = []
|
||||
has_document: bool = False
|
||||
|
||||
|
||||
# ── Follow ────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -15,6 +15,7 @@ celery_app = Celery(
|
||||
"app.workers.news_fetcher",
|
||||
"app.workers.trend_scorer",
|
||||
"app.workers.member_interest",
|
||||
"app.workers.notification_dispatcher",
|
||||
],
|
||||
)
|
||||
|
||||
@@ -37,6 +38,7 @@ celery_app.conf.update(
|
||||
"app.workers.news_fetcher.*": {"queue": "news"},
|
||||
"app.workers.trend_scorer.*": {"queue": "news"},
|
||||
"app.workers.member_interest.*": {"queue": "news"},
|
||||
"app.workers.notification_dispatcher.*": {"queue": "polling"},
|
||||
},
|
||||
task_queues=[
|
||||
Queue("polling"),
|
||||
@@ -72,5 +74,9 @@ celery_app.conf.update(
|
||||
"task": "app.workers.congress_poller.fetch_actions_for_active_bills",
|
||||
"schedule": crontab(hour=4, minute=0), # 4 AM UTC, after trend + member scoring
|
||||
},
|
||||
"dispatch-notifications": {
|
||||
"task": "app.workers.notification_dispatcher.dispatch_notifications",
|
||||
"schedule": crontab(minute="*/5"), # Every 5 minutes
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
@@ -300,17 +300,95 @@ def fetch_actions_for_active_bills(self):
|
||||
def _update_bill_if_changed(db, existing: Bill, parsed: dict) -> bool:
|
||||
"""Update bill fields if anything has changed. Returns True if updated."""
|
||||
changed = False
|
||||
dirty = False
|
||||
|
||||
# Meaningful change fields — trigger document + action fetch when updated
|
||||
track_fields = ["title", "short_title", "latest_action_date", "latest_action_text", "status"]
|
||||
for field in track_fields:
|
||||
new_val = parsed.get(field)
|
||||
if new_val and getattr(existing, field) != new_val:
|
||||
setattr(existing, field, new_val)
|
||||
changed = True
|
||||
dirty = True
|
||||
|
||||
# Static fields — only fill in if currently null; no change trigger needed
|
||||
fill_null_fields = ["introduced_date", "congress_url", "chamber"]
|
||||
for field in fill_null_fields:
|
||||
new_val = parsed.get(field)
|
||||
if new_val and getattr(existing, field) is None:
|
||||
setattr(existing, field, new_val)
|
||||
dirty = True
|
||||
|
||||
if changed:
|
||||
existing.last_checked_at = datetime.now(timezone.utc)
|
||||
if dirty:
|
||||
db.commit()
|
||||
# Check for new text versions and sync actions now that the bill has changed
|
||||
if changed:
|
||||
from app.workers.document_fetcher import fetch_bill_documents
|
||||
fetch_bill_documents.delay(existing.bill_id)
|
||||
fetch_bill_actions.delay(existing.bill_id)
|
||||
return changed
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.workers.congress_poller.backfill_bill_metadata")
|
||||
def backfill_bill_metadata(self):
|
||||
"""
|
||||
Find bills with null introduced_date (or other static fields) and
|
||||
re-fetch their detail from Congress.gov to fill in the missing values.
|
||||
No document or LLM calls — metadata only.
|
||||
"""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
from sqlalchemy import text as sa_text
|
||||
rows = db.execute(sa_text("""
|
||||
SELECT bill_id, congress_number, bill_type, bill_number
|
||||
FROM bills
|
||||
WHERE introduced_date IS NULL
|
||||
OR congress_url IS NULL
|
||||
OR chamber IS NULL
|
||||
""")).fetchall()
|
||||
|
||||
updated = 0
|
||||
skipped = 0
|
||||
for row in rows:
|
||||
try:
|
||||
detail = congress_api.get_bill_detail(
|
||||
row.congress_number, row.bill_type, row.bill_number
|
||||
)
|
||||
bill_data = detail.get("bill", {})
|
||||
parsed = congress_api.parse_bill_from_api(
|
||||
{
|
||||
"type": row.bill_type,
|
||||
"number": row.bill_number,
|
||||
"introducedDate": bill_data.get("introducedDate"),
|
||||
"title": bill_data.get("title"),
|
||||
"shortTitle": bill_data.get("shortTitle"),
|
||||
"latestAction": bill_data.get("latestAction") or {},
|
||||
},
|
||||
row.congress_number,
|
||||
)
|
||||
bill = db.get(Bill, row.bill_id)
|
||||
if not bill:
|
||||
skipped += 1
|
||||
continue
|
||||
fill_null_fields = ["introduced_date", "congress_url", "chamber", "title", "short_title"]
|
||||
dirty = False
|
||||
for field in fill_null_fields:
|
||||
new_val = parsed.get(field)
|
||||
if new_val and getattr(bill, field) is None:
|
||||
setattr(bill, field, new_val)
|
||||
dirty = True
|
||||
if dirty:
|
||||
db.commit()
|
||||
updated += 1
|
||||
else:
|
||||
skipped += 1
|
||||
time.sleep(0.2) # ~300 req/min — well under the 5k/hr limit
|
||||
except Exception as exc:
|
||||
logger.warning(f"backfill_bill_metadata: failed for {row.bill_id}: {exc}")
|
||||
skipped += 1
|
||||
|
||||
logger.info(f"backfill_bill_metadata: {updated} updated, {skipped} skipped")
|
||||
return {"updated": updated, "skipped": skipped}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -199,3 +199,55 @@ def backfill_brief_citations(self):
|
||||
return {"total": total, "queued": queued, "skipped": skipped}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.workers.llm_processor.resume_pending_analysis")
|
||||
def resume_pending_analysis(self):
|
||||
"""
|
||||
Two-pass backfill for bills missing analysis:
|
||||
|
||||
Pass 1 — Documents with no brief (LLM tasks failed/timed out):
|
||||
Find BillDocuments that have raw_text but no BillBrief, re-queue LLM.
|
||||
|
||||
Pass 2 — Bills with no document at all:
|
||||
Find Bills with no BillDocument, re-queue document fetch (which will
|
||||
then chain into LLM if text is available on GovInfo).
|
||||
"""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
# Pass 1: docs with raw_text but no brief
|
||||
docs_no_brief = db.execute(text("""
|
||||
SELECT bd.id
|
||||
FROM bill_documents bd
|
||||
LEFT JOIN bill_briefs bb ON bb.document_id = bd.id
|
||||
WHERE bb.id IS NULL AND bd.raw_text IS NOT NULL
|
||||
""")).fetchall()
|
||||
|
||||
queued_llm = 0
|
||||
for row in docs_no_brief:
|
||||
process_document_with_llm.delay(row.id)
|
||||
queued_llm += 1
|
||||
time.sleep(0.1)
|
||||
|
||||
# Pass 2: bills with no document at all
|
||||
bills_no_doc = db.execute(text("""
|
||||
SELECT b.bill_id
|
||||
FROM bills b
|
||||
LEFT JOIN bill_documents bd ON bd.bill_id = b.bill_id
|
||||
WHERE bd.id IS NULL
|
||||
""")).fetchall()
|
||||
|
||||
queued_fetch = 0
|
||||
from app.workers.document_fetcher import fetch_bill_documents
|
||||
for row in bills_no_doc:
|
||||
fetch_bill_documents.delay(row.bill_id)
|
||||
queued_fetch += 1
|
||||
time.sleep(0.1)
|
||||
|
||||
logger.info(
|
||||
f"resume_pending_analysis: {queued_llm} LLM tasks queued, "
|
||||
f"{queued_fetch} document fetch tasks queued"
|
||||
)
|
||||
return {"queued_llm": queued_llm, "queued_fetch": queued_fetch}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import { use, useEffect, useRef } from "react";
|
||||
import Link from "next/link";
|
||||
import { ArrowLeft, ExternalLink, User } from "lucide-react";
|
||||
import { ArrowLeft, ExternalLink, FileX, User } from "lucide-react";
|
||||
import { useBill, useBillNews, useBillTrend } from "@/lib/hooks/useBills";
|
||||
import { BriefPanel } from "@/components/bills/BriefPanel";
|
||||
import { ActionTimeline } from "@/components/bills/ActionTimeline";
|
||||
@@ -82,10 +82,12 @@ export default function BillDetailPage({ params }: { params: Promise<{ id: strin
|
||||
{bill.sponsor.state && <span>{bill.sponsor.state}</span>}
|
||||
</div>
|
||||
)}
|
||||
<p className="text-xs text-muted-foreground mt-1">
|
||||
Introduced: {formatDate(bill.introduced_date)}
|
||||
<p className="text-xs text-muted-foreground mt-1 flex items-center gap-3 flex-wrap">
|
||||
{bill.introduced_date && (
|
||||
<span>Introduced: {formatDate(bill.introduced_date)}</span>
|
||||
)}
|
||||
{bill.congress_url && (
|
||||
<a href={bill.congress_url} target="_blank" rel="noopener noreferrer" className="ml-3 hover:text-primary transition-colors">
|
||||
<a href={bill.congress_url} target="_blank" rel="noopener noreferrer" className="hover:text-primary transition-colors">
|
||||
congress.gov <ExternalLink className="w-3 h-3 inline" />
|
||||
</a>
|
||||
)}
|
||||
@@ -97,7 +99,39 @@ export default function BillDetailPage({ params }: { params: Promise<{ id: strin
|
||||
{/* Content grid */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-3 gap-4 md:gap-6">
|
||||
<div className="md:col-span-2 space-y-6">
|
||||
{bill.briefs.length > 0 ? (
|
||||
<BriefPanel briefs={bill.briefs} />
|
||||
) : bill.has_document ? (
|
||||
<div className="bg-card border border-border rounded-lg p-6 text-center space-y-2">
|
||||
<p className="text-sm font-medium text-muted-foreground">Analysis pending</p>
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Bill text was retrieved but has not yet been analyzed. Check back shortly.
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="bg-card border border-border rounded-lg p-6 space-y-3">
|
||||
<div className="flex items-center gap-2 text-muted-foreground">
|
||||
<FileX className="w-4 h-4 shrink-0" />
|
||||
<span className="text-sm font-medium">No bill text published</span>
|
||||
</div>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
As of {new Date().toLocaleDateString("en-US", { month: "long", day: "numeric", year: "numeric" })},{" "}
|
||||
no official text has been received for{" "}
|
||||
<span className="font-medium">{billLabel(bill.bill_type, bill.bill_number)}</span>.
|
||||
Analysis will be generated automatically once text is published on Congress.gov.
|
||||
</p>
|
||||
{bill.congress_url && (
|
||||
<a
|
||||
href={bill.congress_url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 text-xs text-primary hover:underline"
|
||||
>
|
||||
Check status on Congress.gov <ExternalLink className="w-3 h-3" />
|
||||
</a>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<ActionTimeline actions={bill.actions} />
|
||||
</div>
|
||||
<div className="space-y-4">
|
||||
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
Copy,
|
||||
Rss,
|
||||
} from "lucide-react";
|
||||
import { settingsAPI, adminAPI, notificationsAPI, type AdminUser, type LLMModel } from "@/lib/api";
|
||||
import { settingsAPI, adminAPI, notificationsAPI, type AdminUser, type LLMModel, type ApiHealthResult } from "@/lib/api";
|
||||
import { useAuthStore } from "@/stores/authStore";
|
||||
|
||||
const LLM_PROVIDERS = [
|
||||
@@ -47,6 +47,18 @@ export default function SettingsPage() {
|
||||
refetchInterval: 30_000,
|
||||
});
|
||||
|
||||
const [healthTesting, setHealthTesting] = useState(false);
|
||||
const [healthData, setHealthData] = useState<Record<string, ApiHealthResult> | null>(null);
|
||||
const testApiHealth = async () => {
|
||||
setHealthTesting(true);
|
||||
try {
|
||||
const result = await adminAPI.getApiHealth();
|
||||
setHealthData(result as unknown as Record<string, ApiHealthResult>);
|
||||
} finally {
|
||||
setHealthTesting(false);
|
||||
}
|
||||
};
|
||||
|
||||
const { data: users, isLoading: usersLoading } = useQuery({
|
||||
queryKey: ["admin-users"],
|
||||
queryFn: () => adminAPI.listUsers(),
|
||||
@@ -167,34 +179,16 @@ export default function SettingsPage() {
|
||||
{/* Analysis Status */}
|
||||
<section className="bg-card border border-border rounded-lg p-6 space-y-4">
|
||||
<h2 className="font-semibold flex items-center gap-2">
|
||||
<BarChart3 className="w-4 h-4" /> Analysis Status
|
||||
<BarChart3 className="w-4 h-4" /> Bill Pipeline
|
||||
<span className="text-xs text-muted-foreground font-normal ml-auto">refreshes every 30s</span>
|
||||
</h2>
|
||||
{stats ? (
|
||||
<>
|
||||
<div className="grid grid-cols-1 sm:grid-cols-3 gap-4">
|
||||
<div className="bg-muted/50 rounded-lg p-3 text-center">
|
||||
<FileText className="w-4 h-4 mx-auto mb-1 text-muted-foreground" />
|
||||
<div className="text-xl font-bold">{stats.total_bills.toLocaleString()}</div>
|
||||
<div className="text-xs text-muted-foreground">Total Bills</div>
|
||||
</div>
|
||||
<div className="bg-muted/50 rounded-lg p-3 text-center">
|
||||
<FileText className="w-4 h-4 mx-auto mb-1 text-blue-500" />
|
||||
<div className="text-xl font-bold">{stats.docs_fetched.toLocaleString()}</div>
|
||||
<div className="text-xs text-muted-foreground">Docs Fetched</div>
|
||||
</div>
|
||||
<div className="bg-muted/50 rounded-lg p-3 text-center">
|
||||
<Brain className="w-4 h-4 mx-auto mb-1 text-green-500" />
|
||||
<div className="text-xl font-bold">{stats.briefs_generated.toLocaleString()}</div>
|
||||
<div className="text-xs text-muted-foreground">Briefs Generated</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Progress bar */}
|
||||
<div className="space-y-1">
|
||||
<div className="flex justify-between text-xs text-muted-foreground">
|
||||
<span>{stats.full_briefs} full · {stats.amendment_briefs} amendments</span>
|
||||
<span>{pct}% analyzed · {stats.remaining.toLocaleString()} remaining</span>
|
||||
<span>{stats.briefs_generated.toLocaleString()} analyzed ({stats.full_briefs} full · {stats.amendment_briefs} amendments)</span>
|
||||
<span>{pct}% of {stats.total_bills.toLocaleString()} bills</span>
|
||||
</div>
|
||||
<div className="h-2 bg-muted rounded-full overflow-hidden">
|
||||
<div
|
||||
@@ -202,12 +196,35 @@ export default function SettingsPage() {
|
||||
style={{ width: `${pct}%` }}
|
||||
/>
|
||||
</div>
|
||||
{stats.uncited_briefs > 0 && (
|
||||
<p className="text-xs text-amber-600 dark:text-amber-400">
|
||||
⚠ {stats.uncited_briefs.toLocaleString()} brief{stats.uncited_briefs !== 1 ? "s" : ""} missing citations — run Backfill Citations to fix
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Pipeline breakdown table */}
|
||||
<div className="divide-y divide-border text-sm">
|
||||
{[
|
||||
{ label: "Total bills tracked", value: stats.total_bills, color: "text-foreground", icon: "📋" },
|
||||
{ label: "Text published on Congress.gov", value: stats.docs_fetched, color: "text-blue-600 dark:text-blue-400", icon: "📄" },
|
||||
{ label: "No text published yet", value: stats.no_text_bills, color: "text-muted-foreground", icon: "⏳", note: "Normal — bill text appears after committee markup" },
|
||||
{ label: "AI briefs generated", value: stats.briefs_generated, color: "text-green-600 dark:text-green-400", icon: "✅" },
|
||||
{ label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined },
|
||||
{ label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined },
|
||||
].map(({ label, value, color, icon, note, action }) => (
|
||||
<div key={label} className="flex items-center justify-between py-2.5 gap-3">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
<span className="text-base leading-none shrink-0">{icon}</span>
|
||||
<div>
|
||||
<span className="text-sm">{label}</span>
|
||||
{note && <p className="text-xs text-muted-foreground mt-0.5">{note}</p>}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-3 shrink-0">
|
||||
<span className={`font-semibold tabular-nums ${color}`}>{value.toLocaleString()}</span>
|
||||
{action && (
|
||||
<span className="text-xs text-muted-foreground">→ run {action}</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">Loading stats...</p>
|
||||
@@ -560,50 +577,171 @@ export default function SettingsPage() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* API Health */}
|
||||
<section className="bg-card border border-border rounded-lg p-6 space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<h2 className="font-semibold">External API Health</h2>
|
||||
<button
|
||||
onClick={testApiHealth}
|
||||
disabled={healthTesting}
|
||||
className="flex items-center gap-2 px-3 py-1.5 text-sm bg-muted hover:bg-accent rounded-md transition-colors disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw className={`w-3.5 h-3.5 ${healthTesting ? "animate-spin" : ""}`} />
|
||||
{healthTesting ? "Testing…" : "Run Tests"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{healthData ? (
|
||||
<div className="divide-y divide-border">
|
||||
{[
|
||||
{ key: "congress_gov", label: "Congress.gov API" },
|
||||
{ key: "govinfo", label: "GovInfo API" },
|
||||
{ key: "newsapi", label: "NewsAPI.org" },
|
||||
{ key: "google_news", label: "Google News RSS" },
|
||||
].map(({ key, label }) => {
|
||||
const r = healthData[key];
|
||||
if (!r) return null;
|
||||
return (
|
||||
<div key={key} className="flex items-start justify-between py-3 gap-4">
|
||||
<div>
|
||||
<div className="text-sm font-medium">{label}</div>
|
||||
<div className={`text-xs mt-0.5 ${
|
||||
r.status === "ok" ? "text-green-600 dark:text-green-400"
|
||||
: r.status === "skipped" ? "text-muted-foreground"
|
||||
: "text-red-600 dark:text-red-400"
|
||||
}`}>
|
||||
{r.detail}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center gap-2 shrink-0">
|
||||
{r.latency_ms !== undefined && (
|
||||
<span className="text-xs text-muted-foreground">{r.latency_ms}ms</span>
|
||||
)}
|
||||
{r.status === "ok" && <CheckCircle className="w-4 h-4 text-green-500" />}
|
||||
{r.status === "error" && <XCircle className="w-4 h-4 text-red-500" />}
|
||||
{r.status === "skipped" && <span className="text-xs text-muted-foreground">—</span>}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
) : (
|
||||
<p className="text-sm text-muted-foreground">
|
||||
Click Run Tests to check connectivity to each external data source.
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
|
||||
{/* Manual Controls */}
|
||||
<section className="bg-card border border-border rounded-lg p-6 space-y-4">
|
||||
<h2 className="font-semibold">Manual Controls</h2>
|
||||
<div className="flex flex-wrap gap-3">
|
||||
<div className="divide-y divide-border">
|
||||
{([
|
||||
{
|
||||
key: "poll",
|
||||
name: "Trigger Poll",
|
||||
description: "Check Congress.gov for newly introduced or updated bills. Runs automatically on a schedule — use this to force an immediate sync.",
|
||||
fn: adminAPI.triggerPoll,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "members",
|
||||
name: "Sync Members",
|
||||
description: "Refresh all member profiles from Congress.gov including biography, current term, leadership roles, and contact information.",
|
||||
fn: adminAPI.triggerMemberSync,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "trends",
|
||||
name: "Calculate Trends",
|
||||
description: "Score bill and member newsworthiness by counting recent news headlines and Google search interest. Updates the trend charts.",
|
||||
fn: adminAPI.triggerTrendScores,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "actions",
|
||||
name: "Fetch Bill Actions",
|
||||
description: "Download the full legislative history (votes, referrals, amendments) for recently active bills and populate the timeline view.",
|
||||
fn: adminAPI.triggerFetchActions,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "sponsors",
|
||||
name: "Backfill Sponsors",
|
||||
description: "Link bill sponsors that weren't captured during the initial import. Safe to re-run — skips bills that already have a sponsor.",
|
||||
fn: adminAPI.backfillSponsors,
|
||||
status: stats ? (stats.bills_missing_sponsor > 0 ? "needed" : "ok") : "on-demand",
|
||||
count: stats?.bills_missing_sponsor,
|
||||
countLabel: "bills missing sponsor",
|
||||
},
|
||||
{
|
||||
key: "metadata",
|
||||
name: "Backfill Dates & Links",
|
||||
description: "Fill in missing introduced dates, chamber assignments, and congress.gov links by re-fetching bill detail from Congress.gov.",
|
||||
fn: adminAPI.backfillMetadata,
|
||||
status: stats ? (stats.bills_missing_metadata > 0 ? "needed" : "ok") : "on-demand",
|
||||
count: stats?.bills_missing_metadata,
|
||||
countLabel: "bills missing metadata",
|
||||
},
|
||||
{
|
||||
key: "citations",
|
||||
name: "Backfill Citations",
|
||||
description: "Regenerate AI briefs that were created before inline source citations were added. Deletes the old brief and re-runs LLM analysis using the already-stored bill text — no new Congress.gov calls.",
|
||||
fn: adminAPI.backfillCitations,
|
||||
status: stats ? (stats.uncited_briefs > 0 ? "needed" : "ok") : "on-demand",
|
||||
count: stats?.uncited_briefs,
|
||||
countLabel: "briefs need regeneration",
|
||||
},
|
||||
{
|
||||
key: "resume",
|
||||
name: "Resume Analysis",
|
||||
description: "Restart AI brief generation for bills where processing stalled or failed (e.g. after an LLM quota outage). Also re-queues document fetching for bills that have no text yet.",
|
||||
fn: adminAPI.resumeAnalysis,
|
||||
status: stats ? (stats.pending_llm > 0 ? "needed" : "ok") : "on-demand",
|
||||
count: stats?.pending_llm,
|
||||
countLabel: "bills pending analysis",
|
||||
},
|
||||
] as Array<{
|
||||
key: string;
|
||||
name: string;
|
||||
description: string;
|
||||
fn: () => Promise<{ task_id: string }>;
|
||||
status: "ok" | "needed" | "on-demand";
|
||||
count?: number;
|
||||
countLabel?: string;
|
||||
}>).map(({ key, name, description, fn, status, count, countLabel }) => (
|
||||
<div key={key} className="flex items-start gap-3 py-3.5">
|
||||
<div className={`w-2.5 h-2.5 rounded-full mt-1 shrink-0 ${
|
||||
status === "ok" ? "bg-green-500"
|
||||
: status === "needed" ? "bg-red-500"
|
||||
: "bg-border"
|
||||
}`} />
|
||||
<div className="flex-1 min-w-0 space-y-0.5">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-sm font-medium">{name}</span>
|
||||
{status === "ok" && (
|
||||
<span className="text-xs text-green-600 dark:text-green-400">✓ Up to date</span>
|
||||
)}
|
||||
{status === "needed" && count !== undefined && count > 0 && (
|
||||
<span className="text-xs text-red-600 dark:text-red-400">
|
||||
⚠ {count.toLocaleString()} {countLabel}
|
||||
</span>
|
||||
)}
|
||||
{taskIds[key] && (
|
||||
<span className="text-xs text-muted-foreground">queued ✓</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-xs text-muted-foreground leading-relaxed">{description}</p>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => trigger("poll", adminAPI.triggerPoll)}
|
||||
className="flex items-center gap-2 px-4 py-2 text-sm bg-muted hover:bg-accent rounded-md transition-colors"
|
||||
onClick={() => trigger(key, fn)}
|
||||
className="shrink-0 px-3 py-1.5 text-xs bg-muted hover:bg-accent rounded-md transition-colors font-medium"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" /> Trigger Poll
|
||||
</button>
|
||||
<button
|
||||
onClick={() => trigger("members", adminAPI.triggerMemberSync)}
|
||||
className="flex items-center gap-2 px-4 py-2 text-sm bg-muted hover:bg-accent rounded-md transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" /> Sync Members
|
||||
</button>
|
||||
<button
|
||||
onClick={() => trigger("trends", adminAPI.triggerTrendScores)}
|
||||
className="flex items-center gap-2 px-4 py-2 text-sm bg-muted hover:bg-accent rounded-md transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" /> Calculate Trends
|
||||
</button>
|
||||
<button
|
||||
onClick={() => trigger("sponsors", adminAPI.backfillSponsors)}
|
||||
className="flex items-center gap-2 px-4 py-2 text-sm bg-muted hover:bg-accent rounded-md transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" /> Backfill Sponsors
|
||||
</button>
|
||||
<button
|
||||
onClick={() => trigger("citations", adminAPI.backfillCitations)}
|
||||
className="flex items-center gap-2 px-4 py-2 text-sm bg-amber-100 text-amber-800 hover:bg-amber-200 dark:bg-amber-900/30 dark:text-amber-300 dark:hover:bg-amber-900/50 rounded-md transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" /> Backfill Citations
|
||||
</button>
|
||||
<button
|
||||
onClick={() => trigger("actions", adminAPI.triggerFetchActions)}
|
||||
className="flex items-center gap-2 px-4 py-2 text-sm bg-muted hover:bg-accent rounded-md transition-colors"
|
||||
>
|
||||
<RefreshCw className="w-3.5 h-3.5" /> Fetch Bill Actions
|
||||
Run
|
||||
</button>
|
||||
</div>
|
||||
{Object.entries(taskIds).map(([name, id]) => (
|
||||
<p key={name} className="text-xs text-muted-foreground">{name}: task {id} queued</p>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -137,6 +137,19 @@ export interface AdminUser {
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export interface ApiHealthResult {
|
||||
status: "ok" | "error" | "skipped";
|
||||
detail: string;
|
||||
latency_ms?: number;
|
||||
}
|
||||
|
||||
export interface ApiHealth {
|
||||
congress_gov: ApiHealthResult;
|
||||
govinfo: ApiHealthResult;
|
||||
newsapi: ApiHealthResult;
|
||||
google_news: ApiHealthResult;
|
||||
}
|
||||
|
||||
export interface AnalysisStats {
|
||||
total_bills: number;
|
||||
docs_fetched: number;
|
||||
@@ -144,6 +157,10 @@ export interface AnalysisStats {
|
||||
full_briefs: number;
|
||||
amendment_briefs: number;
|
||||
uncited_briefs: number;
|
||||
no_text_bills: number;
|
||||
pending_llm: number;
|
||||
bills_missing_sponsor: number;
|
||||
bills_missing_metadata: number;
|
||||
remaining: number;
|
||||
}
|
||||
|
||||
@@ -182,6 +199,12 @@ export const adminAPI = {
|
||||
apiClient.post("/api/admin/backfill-citations").then((r) => r.data),
|
||||
triggerFetchActions: () =>
|
||||
apiClient.post("/api/admin/trigger-fetch-actions").then((r) => r.data),
|
||||
backfillMetadata: () =>
|
||||
apiClient.post("/api/admin/backfill-metadata").then((r) => r.data),
|
||||
resumeAnalysis: () =>
|
||||
apiClient.post("/api/admin/resume-analysis").then((r) => r.data),
|
||||
getApiHealth: () =>
|
||||
apiClient.get<ApiHealth>("/api/admin/api-health").then((r) => r.data),
|
||||
getTaskStatus: (taskId: string) =>
|
||||
apiClient.get(`/api/admin/task-status/${taskId}`).then((r) => r.data),
|
||||
};
|
||||
|
||||
@@ -123,6 +123,7 @@ export interface BillDetail extends Bill {
|
||||
news_articles: NewsArticle[];
|
||||
trend_scores: TrendScore[];
|
||||
briefs: BriefSchema[];
|
||||
has_document: boolean;
|
||||
}
|
||||
|
||||
export interface PaginatedResponse<T> {
|
||||
@@ -153,3 +154,10 @@ export interface SettingsData {
|
||||
newsapi_enabled: boolean;
|
||||
pytrends_enabled: boolean;
|
||||
}
|
||||
|
||||
export interface NotificationSettings {
|
||||
ntfy_topic_url: string;
|
||||
ntfy_token: string;
|
||||
ntfy_enabled: boolean;
|
||||
rss_token: string | null;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user