feat: admin panel — new triggers, member/vote stats, health checks
Backend: - /trigger-fetch-news, /trigger-fetch-votes, /trigger-member-trend-scores endpoints - Stats: total_members, house_count, senate_count, members_missing_chamber, total_votes, stanced_bills_total, stanced_bills_with_votes - API health: redis, smtp, pytrends tests added Frontend: - Fetch News, Fetch Votes, Member Trend Scores buttons in recurring tasks - Fetch Votes shows "needed" badge when stanced bills are missing votes - Stats table: member sync breakdown + vote/alignment coverage row - Health panel: Redis, SMTP, pytrends rows Authored by: Jack Levy
This commit is contained in:
Jack Levy
@@ -5,7 +5,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from app.core.dependencies import get_current_admin
|
||||
from app.database import get_db
|
||||
from app.models import Bill, BillBrief, BillDocument, Follow
|
||||
from app.models.member import Member
|
||||
from app.models.user import User
|
||||
from app.models.vote import BillVote
|
||||
from app.schemas.schemas import UserResponse
|
||||
|
||||
router = APIRouter()
|
||||
@@ -134,6 +136,30 @@ async def get_stats(
|
||||
bills_missing_actions = (await db.execute(
|
||||
text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL")
|
||||
)).scalar()
|
||||
# Member sync health
|
||||
total_members = (await db.execute(select(func.count()).select_from(Member))).scalar()
|
||||
house_count = (await db.execute(
|
||||
select(func.count()).select_from(Member).where(Member.chamber == "House of Representatives")
|
||||
)).scalar()
|
||||
senate_count = (await db.execute(
|
||||
select(func.count()).select_from(Member).where(Member.chamber == "Senate")
|
||||
)).scalar()
|
||||
members_missing_chamber = (await db.execute(
|
||||
select(func.count()).select_from(Member).where(Member.chamber.is_(None))
|
||||
)).scalar()
|
||||
# Vote / alignment coverage
|
||||
total_votes = (await db.execute(select(func.count()).select_from(BillVote))).scalar()
|
||||
stanced_row = (await db.execute(text("""
|
||||
SELECT
|
||||
COUNT(DISTINCT f.follow_value) AS total,
|
||||
COUNT(DISTINCT bv.bill_id) AS with_votes
|
||||
FROM follows f
|
||||
LEFT JOIN bill_votes bv ON bv.bill_id = f.follow_value
|
||||
WHERE f.follow_type = 'bill'
|
||||
AND f.follow_mode IN ('pocket_veto', 'pocket_boost')
|
||||
"""))).fetchone()
|
||||
stanced_bills_total = stanced_row.total if stanced_row else 0
|
||||
stanced_bills_with_votes = stanced_row.with_votes if stanced_row else 0
|
||||
# Cited brief points (objects) that have no label yet
|
||||
unlabeled_briefs = (await db.execute(
|
||||
text("""
|
||||
@@ -165,6 +191,13 @@ async def get_stats(
|
||||
"bills_missing_actions": bills_missing_actions,
|
||||
"unlabeled_briefs": unlabeled_briefs,
|
||||
"remaining": total_bills - total_briefs,
|
||||
"total_members": total_members,
|
||||
"house_count": house_count,
|
||||
"senate_count": senate_count,
|
||||
"members_missing_chamber": members_missing_chamber,
|
||||
"total_votes": total_votes,
|
||||
"stanced_bills_total": stanced_bills_total,
|
||||
"stanced_bills_with_votes": stanced_bills_with_votes,
|
||||
}
|
||||
|
||||
|
||||
@@ -277,6 +310,27 @@ async def trigger_trend_scores(current_user: User = Depends(get_current_admin)):
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/trigger-fetch-news")
|
||||
async def trigger_fetch_news(current_user: User = Depends(get_current_admin)):
|
||||
from app.workers.news_fetcher import fetch_news_for_active_bills
|
||||
task = fetch_news_for_active_bills.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/trigger-fetch-votes")
|
||||
async def trigger_fetch_votes(current_user: User = Depends(get_current_admin)):
|
||||
from app.workers.vote_fetcher import fetch_votes_for_stanced_bills
|
||||
task = fetch_votes_for_stanced_bills.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/trigger-member-trend-scores")
|
||||
async def trigger_member_trend_scores(current_user: User = Depends(get_current_admin)):
|
||||
from app.workers.member_interest import calculate_all_member_trend_scores
|
||||
task = calculate_all_member_trend_scores.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/bills/{bill_id}/reprocess")
|
||||
async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_admin)):
|
||||
"""Queue document and action fetches for a specific bill. Useful for debugging."""
|
||||
@@ -340,9 +394,12 @@ async def api_health(current_user: User = Depends(get_current_admin)):
|
||||
asyncio.to_thread(_test_newsapi),
|
||||
asyncio.to_thread(_test_gnews),
|
||||
asyncio.to_thread(_test_rep_lookup),
|
||||
asyncio.to_thread(_test_redis),
|
||||
asyncio.to_thread(_test_smtp),
|
||||
asyncio.to_thread(_test_pytrends),
|
||||
return_exceptions=True,
|
||||
)
|
||||
keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup"]
|
||||
keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup", "redis", "smtp", "pytrends"]
|
||||
return {
|
||||
k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)}
|
||||
for k, r in zip(keys, results)
|
||||
@@ -495,6 +552,62 @@ def _test_rep_lookup() -> dict:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
def _test_redis() -> dict:
|
||||
from app.config import settings
|
||||
import redis as redis_lib
|
||||
def _call():
|
||||
client = redis_lib.from_url(settings.REDIS_URL, socket_timeout=5)
|
||||
client.ping()
|
||||
info = client.info("server")
|
||||
version = info.get("redis_version", "?")
|
||||
used_mb = round(client.info("memory").get("used_memory", 0) / 1024 / 1024, 1)
|
||||
return {"status": "ok", "detail": f"Redis {version} — {used_mb} MB used"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
def _test_smtp() -> dict:
|
||||
from app.config import settings
|
||||
if not settings.SMTP_HOST:
|
||||
return {"status": "skipped", "detail": "SMTP_HOST not configured"}
|
||||
def _call():
|
||||
import smtplib, ssl
|
||||
if settings.SMTP_STARTTLS:
|
||||
with smtplib.SMTP(settings.SMTP_HOST, settings.SMTP_PORT, timeout=10) as s:
|
||||
s.ehlo()
|
||||
s.starttls(context=ssl.create_default_context())
|
||||
s.ehlo()
|
||||
else:
|
||||
with smtplib.SMTP_SSL(settings.SMTP_HOST, settings.SMTP_PORT,
|
||||
context=ssl.create_default_context(), timeout=10) as s:
|
||||
s.ehlo()
|
||||
return {"status": "ok", "detail": f"SMTP connected to {settings.SMTP_HOST}:{settings.SMTP_PORT}"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
def _test_pytrends() -> dict:
|
||||
from app.config import settings
|
||||
if not settings.PYTRENDS_ENABLED:
|
||||
return {"status": "skipped", "detail": "PYTRENDS_ENABLED is false"}
|
||||
def _call():
|
||||
from pytrends.request import TrendReq
|
||||
pt = TrendReq(hl="en-US", tz=360, timeout=(5, 15))
|
||||
pt.build_payload(["congress"], timeframe="now 1-d", geo="US")
|
||||
data = pt.interest_over_time()
|
||||
if data is None or data.empty:
|
||||
return {"status": "error", "detail": "pytrends returned empty data — may be rate-limited"}
|
||||
return {"status": "ok", "detail": f"pytrends reachable — {len(data)} data points returned"}
|
||||
try:
|
||||
return _timed(_call)
|
||||
except Exception as exc:
|
||||
return {"status": "error", "detail": str(exc)}
|
||||
|
||||
|
||||
@router.get("/task-status/{task_id}")
|
||||
async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)):
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
@@ -380,6 +380,9 @@ export default function SettingsPage() {
|
||||
{ label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined },
|
||||
{ label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined },
|
||||
{ label: "Briefs with unlabeled points", value: stats.unlabeled_briefs, color: stats.unlabeled_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🏷️", action: stats.unlabeled_briefs > 0 ? "Backfill Labels" : undefined },
|
||||
{ label: "Members synced", value: stats.total_members, color: "text-foreground", icon: "🏛️", note: `${stats.house_count} House · ${stats.senate_count} Senate${stats.members_missing_chamber > 0 ? ` · ⚠️ ${stats.members_missing_chamber} missing chamber` : ""}` },
|
||||
{ label: "Roll-call votes stored", value: stats.total_votes, color: "text-foreground", icon: "🗳️" },
|
||||
{ label: "Stanced bills with votes", value: stats.stanced_bills_with_votes, color: "text-foreground", icon: "⚖️", note: stats.stanced_bills_total > 0 ? `${stats.stanced_bills_with_votes} of ${stats.stanced_bills_total} stanced bills have vote data` : "No stanced bills yet", action: stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "Fetch Votes" : undefined },
|
||||
].map(({ label, value, color, icon, note, action }) => (
|
||||
<div key={label} className="flex items-center justify-between py-2.5 gap-3">
|
||||
<div className="flex items-center gap-2 min-w-0">
|
||||
@@ -688,6 +691,9 @@ export default function SettingsPage() {
|
||||
{ key: "newsapi", label: "NewsAPI.org" },
|
||||
{ key: "google_news", label: "Google News RSS" },
|
||||
{ key: "rep_lookup", label: "Rep Lookup (Nominatim + TIGERweb)" },
|
||||
{ key: "redis", label: "Redis" },
|
||||
{ key: "smtp", label: "SMTP (Email)" },
|
||||
{ key: "pytrends", label: "Google Trends (pytrends)" },
|
||||
].map(({ key, label }) => {
|
||||
const r = healthData[key];
|
||||
if (!r) return null;
|
||||
@@ -826,6 +832,29 @@ export default function SettingsPage() {
|
||||
fn: adminAPI.triggerTrendScores,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "fetch-news",
|
||||
name: "Fetch News",
|
||||
description: "Pull recent news articles for all active bills from NewsAPI and Google News. Runs automatically every 6 hours.",
|
||||
fn: adminAPI.triggerFetchNews,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "fetch-votes",
|
||||
name: "Fetch Votes",
|
||||
description: "Fetch roll-call votes for all bills you have stanced (Pocket Veto or Pocket Boost). Required for Alignment to show data. Re-fetches bills with new activity since the last run.",
|
||||
fn: adminAPI.triggerFetchVotes,
|
||||
status: stats ? (stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "needed" : "on-demand") : "on-demand",
|
||||
count: stats ? stats.stanced_bills_total - stats.stanced_bills_with_votes : undefined,
|
||||
countLabel: "stanced bills missing votes",
|
||||
},
|
||||
{
|
||||
key: "member-trends",
|
||||
name: "Member Trend Scores",
|
||||
description: "Calculate newsworthiness scores for legislators based on recent news coverage. Runs automatically nightly.",
|
||||
fn: adminAPI.triggerMemberTrendScores,
|
||||
status: "on-demand",
|
||||
},
|
||||
{
|
||||
key: "actions",
|
||||
name: "Fetch Bill Actions",
|
||||
|
||||
@@ -208,6 +208,10 @@ export interface ApiHealth {
|
||||
govinfo: ApiHealthResult;
|
||||
newsapi: ApiHealthResult;
|
||||
google_news: ApiHealthResult;
|
||||
rep_lookup: ApiHealthResult;
|
||||
redis: ApiHealthResult;
|
||||
smtp: ApiHealthResult;
|
||||
pytrends: ApiHealthResult;
|
||||
}
|
||||
|
||||
export interface AnalysisStats {
|
||||
@@ -224,6 +228,13 @@ export interface AnalysisStats {
|
||||
bills_missing_actions: number;
|
||||
unlabeled_briefs: number;
|
||||
remaining: number;
|
||||
total_members: number;
|
||||
house_count: number;
|
||||
senate_count: number;
|
||||
members_missing_chamber: number;
|
||||
total_votes: number;
|
||||
stanced_bills_total: number;
|
||||
stanced_bills_with_votes: number;
|
||||
}
|
||||
|
||||
export interface NotificationTestResult {
|
||||
@@ -277,6 +288,12 @@ export const adminAPI = {
|
||||
apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data),
|
||||
triggerTrendScores: () =>
|
||||
apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data),
|
||||
triggerFetchNews: () =>
|
||||
apiClient.post("/api/admin/trigger-fetch-news").then((r) => r.data),
|
||||
triggerFetchVotes: () =>
|
||||
apiClient.post("/api/admin/trigger-fetch-votes").then((r) => r.data),
|
||||
triggerMemberTrendScores: () =>
|
||||
apiClient.post("/api/admin/trigger-member-trend-scores").then((r) => r.data),
|
||||
backfillSponsors: () =>
|
||||
apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data),
|
||||
backfillCitations: () =>
|
||||
|
||||
Reference in New Issue
Block a user