feat: admin panel — new triggers, member/vote stats, health checks

Backend:
- /trigger-fetch-news, /trigger-fetch-votes, /trigger-member-trend-scores endpoints
- Stats: total_members, house_count, senate_count, members_missing_chamber,
  total_votes, stanced_bills_total, stanced_bills_with_votes
- API health: redis, smtp, pytrends tests added

Frontend:
- Fetch News, Fetch Votes, Member Trend Scores buttons in recurring tasks
- Fetch Votes shows "needed" badge when stanced bills are missing votes
- Stats table: member sync breakdown + vote/alignment coverage row
- Health panel: Redis, SMTP, pytrends rows

Authored by: Jack Levy
This commit is contained in:
Jack Levy
2026-03-15 16:50:46 -04:00
parent b952db1806
commit 9f4c9c7a56
3 changed files with 160 additions and 1 deletions

View File

@@ -5,7 +5,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.dependencies import get_current_admin from app.core.dependencies import get_current_admin
from app.database import get_db from app.database import get_db
from app.models import Bill, BillBrief, BillDocument, Follow from app.models import Bill, BillBrief, BillDocument, Follow
from app.models.member import Member
from app.models.user import User from app.models.user import User
from app.models.vote import BillVote
from app.schemas.schemas import UserResponse from app.schemas.schemas import UserResponse
router = APIRouter() router = APIRouter()
@@ -134,6 +136,30 @@ async def get_stats(
bills_missing_actions = (await db.execute( bills_missing_actions = (await db.execute(
text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL") text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL")
)).scalar() )).scalar()
# Member sync health
total_members = (await db.execute(select(func.count()).select_from(Member))).scalar()
house_count = (await db.execute(
select(func.count()).select_from(Member).where(Member.chamber == "House of Representatives")
)).scalar()
senate_count = (await db.execute(
select(func.count()).select_from(Member).where(Member.chamber == "Senate")
)).scalar()
members_missing_chamber = (await db.execute(
select(func.count()).select_from(Member).where(Member.chamber.is_(None))
)).scalar()
# Vote / alignment coverage
total_votes = (await db.execute(select(func.count()).select_from(BillVote))).scalar()
stanced_row = (await db.execute(text("""
SELECT
COUNT(DISTINCT f.follow_value) AS total,
COUNT(DISTINCT bv.bill_id) AS with_votes
FROM follows f
LEFT JOIN bill_votes bv ON bv.bill_id = f.follow_value
WHERE f.follow_type = 'bill'
AND f.follow_mode IN ('pocket_veto', 'pocket_boost')
"""))).fetchone()
stanced_bills_total = stanced_row.total if stanced_row else 0
stanced_bills_with_votes = stanced_row.with_votes if stanced_row else 0
# Cited brief points (objects) that have no label yet # Cited brief points (objects) that have no label yet
unlabeled_briefs = (await db.execute( unlabeled_briefs = (await db.execute(
text(""" text("""
@@ -165,6 +191,13 @@ async def get_stats(
"bills_missing_actions": bills_missing_actions, "bills_missing_actions": bills_missing_actions,
"unlabeled_briefs": unlabeled_briefs, "unlabeled_briefs": unlabeled_briefs,
"remaining": total_bills - total_briefs, "remaining": total_bills - total_briefs,
"total_members": total_members,
"house_count": house_count,
"senate_count": senate_count,
"members_missing_chamber": members_missing_chamber,
"total_votes": total_votes,
"stanced_bills_total": stanced_bills_total,
"stanced_bills_with_votes": stanced_bills_with_votes,
} }
@@ -277,6 +310,27 @@ async def trigger_trend_scores(current_user: User = Depends(get_current_admin)):
return {"task_id": task.id, "status": "queued"} return {"task_id": task.id, "status": "queued"}
@router.post("/trigger-fetch-news")
async def trigger_fetch_news(current_user: User = Depends(get_current_admin)):
from app.workers.news_fetcher import fetch_news_for_active_bills
task = fetch_news_for_active_bills.delay()
return {"task_id": task.id, "status": "queued"}
@router.post("/trigger-fetch-votes")
async def trigger_fetch_votes(current_user: User = Depends(get_current_admin)):
from app.workers.vote_fetcher import fetch_votes_for_stanced_bills
task = fetch_votes_for_stanced_bills.delay()
return {"task_id": task.id, "status": "queued"}
@router.post("/trigger-member-trend-scores")
async def trigger_member_trend_scores(current_user: User = Depends(get_current_admin)):
from app.workers.member_interest import calculate_all_member_trend_scores
task = calculate_all_member_trend_scores.delay()
return {"task_id": task.id, "status": "queued"}
@router.post("/bills/{bill_id}/reprocess") @router.post("/bills/{bill_id}/reprocess")
async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_admin)): async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_admin)):
"""Queue document and action fetches for a specific bill. Useful for debugging.""" """Queue document and action fetches for a specific bill. Useful for debugging."""
@@ -340,9 +394,12 @@ async def api_health(current_user: User = Depends(get_current_admin)):
asyncio.to_thread(_test_newsapi), asyncio.to_thread(_test_newsapi),
asyncio.to_thread(_test_gnews), asyncio.to_thread(_test_gnews),
asyncio.to_thread(_test_rep_lookup), asyncio.to_thread(_test_rep_lookup),
asyncio.to_thread(_test_redis),
asyncio.to_thread(_test_smtp),
asyncio.to_thread(_test_pytrends),
return_exceptions=True, return_exceptions=True,
) )
keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup"] keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup", "redis", "smtp", "pytrends"]
return { return {
k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)} k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)}
for k, r in zip(keys, results) for k, r in zip(keys, results)
@@ -495,6 +552,62 @@ def _test_rep_lookup() -> dict:
return {"status": "error", "detail": str(exc)} return {"status": "error", "detail": str(exc)}
def _test_redis() -> dict:
from app.config import settings
import redis as redis_lib
def _call():
client = redis_lib.from_url(settings.REDIS_URL, socket_timeout=5)
client.ping()
info = client.info("server")
version = info.get("redis_version", "?")
used_mb = round(client.info("memory").get("used_memory", 0) / 1024 / 1024, 1)
return {"status": "ok", "detail": f"Redis {version}{used_mb} MB used"}
try:
return _timed(_call)
except Exception as exc:
return {"status": "error", "detail": str(exc)}
def _test_smtp() -> dict:
from app.config import settings
if not settings.SMTP_HOST:
return {"status": "skipped", "detail": "SMTP_HOST not configured"}
def _call():
import smtplib, ssl
if settings.SMTP_STARTTLS:
with smtplib.SMTP(settings.SMTP_HOST, settings.SMTP_PORT, timeout=10) as s:
s.ehlo()
s.starttls(context=ssl.create_default_context())
s.ehlo()
else:
with smtplib.SMTP_SSL(settings.SMTP_HOST, settings.SMTP_PORT,
context=ssl.create_default_context(), timeout=10) as s:
s.ehlo()
return {"status": "ok", "detail": f"SMTP connected to {settings.SMTP_HOST}:{settings.SMTP_PORT}"}
try:
return _timed(_call)
except Exception as exc:
return {"status": "error", "detail": str(exc)}
def _test_pytrends() -> dict:
from app.config import settings
if not settings.PYTRENDS_ENABLED:
return {"status": "skipped", "detail": "PYTRENDS_ENABLED is false"}
def _call():
from pytrends.request import TrendReq
pt = TrendReq(hl="en-US", tz=360, timeout=(5, 15))
pt.build_payload(["congress"], timeframe="now 1-d", geo="US")
data = pt.interest_over_time()
if data is None or data.empty:
return {"status": "error", "detail": "pytrends returned empty data — may be rate-limited"}
return {"status": "ok", "detail": f"pytrends reachable — {len(data)} data points returned"}
try:
return _timed(_call)
except Exception as exc:
return {"status": "error", "detail": str(exc)}
@router.get("/task-status/{task_id}") @router.get("/task-status/{task_id}")
async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)): async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)):
from app.workers.celery_app import celery_app from app.workers.celery_app import celery_app

View File

@@ -380,6 +380,9 @@ export default function SettingsPage() {
{ label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined }, { label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined },
{ label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined }, { label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined },
{ label: "Briefs with unlabeled points", value: stats.unlabeled_briefs, color: stats.unlabeled_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🏷️", action: stats.unlabeled_briefs > 0 ? "Backfill Labels" : undefined }, { label: "Briefs with unlabeled points", value: stats.unlabeled_briefs, color: stats.unlabeled_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🏷️", action: stats.unlabeled_briefs > 0 ? "Backfill Labels" : undefined },
{ label: "Members synced", value: stats.total_members, color: "text-foreground", icon: "🏛️", note: `${stats.house_count} House · ${stats.senate_count} Senate${stats.members_missing_chamber > 0 ? ` · ⚠️ ${stats.members_missing_chamber} missing chamber` : ""}` },
{ label: "Roll-call votes stored", value: stats.total_votes, color: "text-foreground", icon: "🗳️" },
{ label: "Stanced bills with votes", value: stats.stanced_bills_with_votes, color: "text-foreground", icon: "⚖️", note: stats.stanced_bills_total > 0 ? `${stats.stanced_bills_with_votes} of ${stats.stanced_bills_total} stanced bills have vote data` : "No stanced bills yet", action: stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "Fetch Votes" : undefined },
].map(({ label, value, color, icon, note, action }) => ( ].map(({ label, value, color, icon, note, action }) => (
<div key={label} className="flex items-center justify-between py-2.5 gap-3"> <div key={label} className="flex items-center justify-between py-2.5 gap-3">
<div className="flex items-center gap-2 min-w-0"> <div className="flex items-center gap-2 min-w-0">
@@ -688,6 +691,9 @@ export default function SettingsPage() {
{ key: "newsapi", label: "NewsAPI.org" }, { key: "newsapi", label: "NewsAPI.org" },
{ key: "google_news", label: "Google News RSS" }, { key: "google_news", label: "Google News RSS" },
{ key: "rep_lookup", label: "Rep Lookup (Nominatim + TIGERweb)" }, { key: "rep_lookup", label: "Rep Lookup (Nominatim + TIGERweb)" },
{ key: "redis", label: "Redis" },
{ key: "smtp", label: "SMTP (Email)" },
{ key: "pytrends", label: "Google Trends (pytrends)" },
].map(({ key, label }) => { ].map(({ key, label }) => {
const r = healthData[key]; const r = healthData[key];
if (!r) return null; if (!r) return null;
@@ -826,6 +832,29 @@ export default function SettingsPage() {
fn: adminAPI.triggerTrendScores, fn: adminAPI.triggerTrendScores,
status: "on-demand", status: "on-demand",
}, },
{
key: "fetch-news",
name: "Fetch News",
description: "Pull recent news articles for all active bills from NewsAPI and Google News. Runs automatically every 6 hours.",
fn: adminAPI.triggerFetchNews,
status: "on-demand",
},
{
key: "fetch-votes",
name: "Fetch Votes",
description: "Fetch roll-call votes for all bills you have stanced (Pocket Veto or Pocket Boost). Required for Alignment to show data. Re-fetches bills with new activity since the last run.",
fn: adminAPI.triggerFetchVotes,
status: stats ? (stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "needed" : "on-demand") : "on-demand",
count: stats ? stats.stanced_bills_total - stats.stanced_bills_with_votes : undefined,
countLabel: "stanced bills missing votes",
},
{
key: "member-trends",
name: "Member Trend Scores",
description: "Calculate newsworthiness scores for legislators based on recent news coverage. Runs automatically nightly.",
fn: adminAPI.triggerMemberTrendScores,
status: "on-demand",
},
{ {
key: "actions", key: "actions",
name: "Fetch Bill Actions", name: "Fetch Bill Actions",

View File

@@ -208,6 +208,10 @@ export interface ApiHealth {
govinfo: ApiHealthResult; govinfo: ApiHealthResult;
newsapi: ApiHealthResult; newsapi: ApiHealthResult;
google_news: ApiHealthResult; google_news: ApiHealthResult;
rep_lookup: ApiHealthResult;
redis: ApiHealthResult;
smtp: ApiHealthResult;
pytrends: ApiHealthResult;
} }
export interface AnalysisStats { export interface AnalysisStats {
@@ -224,6 +228,13 @@ export interface AnalysisStats {
bills_missing_actions: number; bills_missing_actions: number;
unlabeled_briefs: number; unlabeled_briefs: number;
remaining: number; remaining: number;
total_members: number;
house_count: number;
senate_count: number;
members_missing_chamber: number;
total_votes: number;
stanced_bills_total: number;
stanced_bills_with_votes: number;
} }
export interface NotificationTestResult { export interface NotificationTestResult {
@@ -277,6 +288,12 @@ export const adminAPI = {
apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data), apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data),
triggerTrendScores: () => triggerTrendScores: () =>
apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data), apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data),
triggerFetchNews: () =>
apiClient.post("/api/admin/trigger-fetch-news").then((r) => r.data),
triggerFetchVotes: () =>
apiClient.post("/api/admin/trigger-fetch-votes").then((r) => r.data),
triggerMemberTrendScores: () =>
apiClient.post("/api/admin/trigger-member-trend-scores").then((r) => r.data),
backfillSponsors: () => backfillSponsors: () =>
apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data), apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data),
backfillCitations: () => backfillCitations: () =>