feat: admin panel — new triggers, member/vote stats, health checks
Backend: - /trigger-fetch-news, /trigger-fetch-votes, /trigger-member-trend-scores endpoints - Stats: total_members, house_count, senate_count, members_missing_chamber, total_votes, stanced_bills_total, stanced_bills_with_votes - API health: redis, smtp, pytrends tests added Frontend: - Fetch News, Fetch Votes, Member Trend Scores buttons in recurring tasks - Fetch Votes shows "needed" badge when stanced bills are missing votes - Stats table: member sync breakdown + vote/alignment coverage row - Health panel: Redis, SMTP, pytrends rows Authored by: Jack Levy
This commit is contained in:
@@ -5,7 +5,9 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from app.core.dependencies import get_current_admin
|
from app.core.dependencies import get_current_admin
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.models import Bill, BillBrief, BillDocument, Follow
|
from app.models import Bill, BillBrief, BillDocument, Follow
|
||||||
|
from app.models.member import Member
|
||||||
from app.models.user import User
|
from app.models.user import User
|
||||||
|
from app.models.vote import BillVote
|
||||||
from app.schemas.schemas import UserResponse
|
from app.schemas.schemas import UserResponse
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
@@ -134,6 +136,30 @@ async def get_stats(
|
|||||||
bills_missing_actions = (await db.execute(
|
bills_missing_actions = (await db.execute(
|
||||||
text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL")
|
text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL")
|
||||||
)).scalar()
|
)).scalar()
|
||||||
|
# Member sync health
|
||||||
|
total_members = (await db.execute(select(func.count()).select_from(Member))).scalar()
|
||||||
|
house_count = (await db.execute(
|
||||||
|
select(func.count()).select_from(Member).where(Member.chamber == "House of Representatives")
|
||||||
|
)).scalar()
|
||||||
|
senate_count = (await db.execute(
|
||||||
|
select(func.count()).select_from(Member).where(Member.chamber == "Senate")
|
||||||
|
)).scalar()
|
||||||
|
members_missing_chamber = (await db.execute(
|
||||||
|
select(func.count()).select_from(Member).where(Member.chamber.is_(None))
|
||||||
|
)).scalar()
|
||||||
|
# Vote / alignment coverage
|
||||||
|
total_votes = (await db.execute(select(func.count()).select_from(BillVote))).scalar()
|
||||||
|
stanced_row = (await db.execute(text("""
|
||||||
|
SELECT
|
||||||
|
COUNT(DISTINCT f.follow_value) AS total,
|
||||||
|
COUNT(DISTINCT bv.bill_id) AS with_votes
|
||||||
|
FROM follows f
|
||||||
|
LEFT JOIN bill_votes bv ON bv.bill_id = f.follow_value
|
||||||
|
WHERE f.follow_type = 'bill'
|
||||||
|
AND f.follow_mode IN ('pocket_veto', 'pocket_boost')
|
||||||
|
"""))).fetchone()
|
||||||
|
stanced_bills_total = stanced_row.total if stanced_row else 0
|
||||||
|
stanced_bills_with_votes = stanced_row.with_votes if stanced_row else 0
|
||||||
# Cited brief points (objects) that have no label yet
|
# Cited brief points (objects) that have no label yet
|
||||||
unlabeled_briefs = (await db.execute(
|
unlabeled_briefs = (await db.execute(
|
||||||
text("""
|
text("""
|
||||||
@@ -165,6 +191,13 @@ async def get_stats(
|
|||||||
"bills_missing_actions": bills_missing_actions,
|
"bills_missing_actions": bills_missing_actions,
|
||||||
"unlabeled_briefs": unlabeled_briefs,
|
"unlabeled_briefs": unlabeled_briefs,
|
||||||
"remaining": total_bills - total_briefs,
|
"remaining": total_bills - total_briefs,
|
||||||
|
"total_members": total_members,
|
||||||
|
"house_count": house_count,
|
||||||
|
"senate_count": senate_count,
|
||||||
|
"members_missing_chamber": members_missing_chamber,
|
||||||
|
"total_votes": total_votes,
|
||||||
|
"stanced_bills_total": stanced_bills_total,
|
||||||
|
"stanced_bills_with_votes": stanced_bills_with_votes,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -277,6 +310,27 @@ async def trigger_trend_scores(current_user: User = Depends(get_current_admin)):
|
|||||||
return {"task_id": task.id, "status": "queued"}
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/trigger-fetch-news")
|
||||||
|
async def trigger_fetch_news(current_user: User = Depends(get_current_admin)):
|
||||||
|
from app.workers.news_fetcher import fetch_news_for_active_bills
|
||||||
|
task = fetch_news_for_active_bills.delay()
|
||||||
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/trigger-fetch-votes")
|
||||||
|
async def trigger_fetch_votes(current_user: User = Depends(get_current_admin)):
|
||||||
|
from app.workers.vote_fetcher import fetch_votes_for_stanced_bills
|
||||||
|
task = fetch_votes_for_stanced_bills.delay()
|
||||||
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/trigger-member-trend-scores")
|
||||||
|
async def trigger_member_trend_scores(current_user: User = Depends(get_current_admin)):
|
||||||
|
from app.workers.member_interest import calculate_all_member_trend_scores
|
||||||
|
task = calculate_all_member_trend_scores.delay()
|
||||||
|
return {"task_id": task.id, "status": "queued"}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/bills/{bill_id}/reprocess")
|
@router.post("/bills/{bill_id}/reprocess")
|
||||||
async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_admin)):
|
async def reprocess_bill(bill_id: str, current_user: User = Depends(get_current_admin)):
|
||||||
"""Queue document and action fetches for a specific bill. Useful for debugging."""
|
"""Queue document and action fetches for a specific bill. Useful for debugging."""
|
||||||
@@ -340,9 +394,12 @@ async def api_health(current_user: User = Depends(get_current_admin)):
|
|||||||
asyncio.to_thread(_test_newsapi),
|
asyncio.to_thread(_test_newsapi),
|
||||||
asyncio.to_thread(_test_gnews),
|
asyncio.to_thread(_test_gnews),
|
||||||
asyncio.to_thread(_test_rep_lookup),
|
asyncio.to_thread(_test_rep_lookup),
|
||||||
|
asyncio.to_thread(_test_redis),
|
||||||
|
asyncio.to_thread(_test_smtp),
|
||||||
|
asyncio.to_thread(_test_pytrends),
|
||||||
return_exceptions=True,
|
return_exceptions=True,
|
||||||
)
|
)
|
||||||
keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup"]
|
keys = ["congress_gov", "govinfo", "newsapi", "google_news", "rep_lookup", "redis", "smtp", "pytrends"]
|
||||||
return {
|
return {
|
||||||
k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)}
|
k: r if isinstance(r, dict) else {"status": "error", "detail": str(r)}
|
||||||
for k, r in zip(keys, results)
|
for k, r in zip(keys, results)
|
||||||
@@ -495,6 +552,62 @@ def _test_rep_lookup() -> dict:
|
|||||||
return {"status": "error", "detail": str(exc)}
|
return {"status": "error", "detail": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _test_redis() -> dict:
|
||||||
|
from app.config import settings
|
||||||
|
import redis as redis_lib
|
||||||
|
def _call():
|
||||||
|
client = redis_lib.from_url(settings.REDIS_URL, socket_timeout=5)
|
||||||
|
client.ping()
|
||||||
|
info = client.info("server")
|
||||||
|
version = info.get("redis_version", "?")
|
||||||
|
used_mb = round(client.info("memory").get("used_memory", 0) / 1024 / 1024, 1)
|
||||||
|
return {"status": "ok", "detail": f"Redis {version} — {used_mb} MB used"}
|
||||||
|
try:
|
||||||
|
return _timed(_call)
|
||||||
|
except Exception as exc:
|
||||||
|
return {"status": "error", "detail": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _test_smtp() -> dict:
|
||||||
|
from app.config import settings
|
||||||
|
if not settings.SMTP_HOST:
|
||||||
|
return {"status": "skipped", "detail": "SMTP_HOST not configured"}
|
||||||
|
def _call():
|
||||||
|
import smtplib, ssl
|
||||||
|
if settings.SMTP_STARTTLS:
|
||||||
|
with smtplib.SMTP(settings.SMTP_HOST, settings.SMTP_PORT, timeout=10) as s:
|
||||||
|
s.ehlo()
|
||||||
|
s.starttls(context=ssl.create_default_context())
|
||||||
|
s.ehlo()
|
||||||
|
else:
|
||||||
|
with smtplib.SMTP_SSL(settings.SMTP_HOST, settings.SMTP_PORT,
|
||||||
|
context=ssl.create_default_context(), timeout=10) as s:
|
||||||
|
s.ehlo()
|
||||||
|
return {"status": "ok", "detail": f"SMTP connected to {settings.SMTP_HOST}:{settings.SMTP_PORT}"}
|
||||||
|
try:
|
||||||
|
return _timed(_call)
|
||||||
|
except Exception as exc:
|
||||||
|
return {"status": "error", "detail": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
|
def _test_pytrends() -> dict:
|
||||||
|
from app.config import settings
|
||||||
|
if not settings.PYTRENDS_ENABLED:
|
||||||
|
return {"status": "skipped", "detail": "PYTRENDS_ENABLED is false"}
|
||||||
|
def _call():
|
||||||
|
from pytrends.request import TrendReq
|
||||||
|
pt = TrendReq(hl="en-US", tz=360, timeout=(5, 15))
|
||||||
|
pt.build_payload(["congress"], timeframe="now 1-d", geo="US")
|
||||||
|
data = pt.interest_over_time()
|
||||||
|
if data is None or data.empty:
|
||||||
|
return {"status": "error", "detail": "pytrends returned empty data — may be rate-limited"}
|
||||||
|
return {"status": "ok", "detail": f"pytrends reachable — {len(data)} data points returned"}
|
||||||
|
try:
|
||||||
|
return _timed(_call)
|
||||||
|
except Exception as exc:
|
||||||
|
return {"status": "error", "detail": str(exc)}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/task-status/{task_id}")
|
@router.get("/task-status/{task_id}")
|
||||||
async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)):
|
async def get_task_status(task_id: str, current_user: User = Depends(get_current_admin)):
|
||||||
from app.workers.celery_app import celery_app
|
from app.workers.celery_app import celery_app
|
||||||
|
|||||||
@@ -380,6 +380,9 @@ export default function SettingsPage() {
|
|||||||
{ label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined },
|
{ label: "Pending LLM analysis", value: stats.pending_llm, color: stats.pending_llm > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🔄", action: stats.pending_llm > 0 ? "Resume Analysis" : undefined },
|
||||||
{ label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined },
|
{ label: "Briefs missing citations", value: stats.uncited_briefs, color: stats.uncited_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "⚠️", action: stats.uncited_briefs > 0 ? "Backfill Citations" : undefined },
|
||||||
{ label: "Briefs with unlabeled points", value: stats.unlabeled_briefs, color: stats.unlabeled_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🏷️", action: stats.unlabeled_briefs > 0 ? "Backfill Labels" : undefined },
|
{ label: "Briefs with unlabeled points", value: stats.unlabeled_briefs, color: stats.unlabeled_briefs > 0 ? "text-amber-600 dark:text-amber-400" : "text-muted-foreground", icon: "🏷️", action: stats.unlabeled_briefs > 0 ? "Backfill Labels" : undefined },
|
||||||
|
{ label: "Members synced", value: stats.total_members, color: "text-foreground", icon: "🏛️", note: `${stats.house_count} House · ${stats.senate_count} Senate${stats.members_missing_chamber > 0 ? ` · ⚠️ ${stats.members_missing_chamber} missing chamber` : ""}` },
|
||||||
|
{ label: "Roll-call votes stored", value: stats.total_votes, color: "text-foreground", icon: "🗳️" },
|
||||||
|
{ label: "Stanced bills with votes", value: stats.stanced_bills_with_votes, color: "text-foreground", icon: "⚖️", note: stats.stanced_bills_total > 0 ? `${stats.stanced_bills_with_votes} of ${stats.stanced_bills_total} stanced bills have vote data` : "No stanced bills yet", action: stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "Fetch Votes" : undefined },
|
||||||
].map(({ label, value, color, icon, note, action }) => (
|
].map(({ label, value, color, icon, note, action }) => (
|
||||||
<div key={label} className="flex items-center justify-between py-2.5 gap-3">
|
<div key={label} className="flex items-center justify-between py-2.5 gap-3">
|
||||||
<div className="flex items-center gap-2 min-w-0">
|
<div className="flex items-center gap-2 min-w-0">
|
||||||
@@ -688,6 +691,9 @@ export default function SettingsPage() {
|
|||||||
{ key: "newsapi", label: "NewsAPI.org" },
|
{ key: "newsapi", label: "NewsAPI.org" },
|
||||||
{ key: "google_news", label: "Google News RSS" },
|
{ key: "google_news", label: "Google News RSS" },
|
||||||
{ key: "rep_lookup", label: "Rep Lookup (Nominatim + TIGERweb)" },
|
{ key: "rep_lookup", label: "Rep Lookup (Nominatim + TIGERweb)" },
|
||||||
|
{ key: "redis", label: "Redis" },
|
||||||
|
{ key: "smtp", label: "SMTP (Email)" },
|
||||||
|
{ key: "pytrends", label: "Google Trends (pytrends)" },
|
||||||
].map(({ key, label }) => {
|
].map(({ key, label }) => {
|
||||||
const r = healthData[key];
|
const r = healthData[key];
|
||||||
if (!r) return null;
|
if (!r) return null;
|
||||||
@@ -826,6 +832,29 @@ export default function SettingsPage() {
|
|||||||
fn: adminAPI.triggerTrendScores,
|
fn: adminAPI.triggerTrendScores,
|
||||||
status: "on-demand",
|
status: "on-demand",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
key: "fetch-news",
|
||||||
|
name: "Fetch News",
|
||||||
|
description: "Pull recent news articles for all active bills from NewsAPI and Google News. Runs automatically every 6 hours.",
|
||||||
|
fn: adminAPI.triggerFetchNews,
|
||||||
|
status: "on-demand",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "fetch-votes",
|
||||||
|
name: "Fetch Votes",
|
||||||
|
description: "Fetch roll-call votes for all bills you have stanced (Pocket Veto or Pocket Boost). Required for Alignment to show data. Re-fetches bills with new activity since the last run.",
|
||||||
|
fn: adminAPI.triggerFetchVotes,
|
||||||
|
status: stats ? (stats.stanced_bills_total > 0 && stats.stanced_bills_with_votes < stats.stanced_bills_total ? "needed" : "on-demand") : "on-demand",
|
||||||
|
count: stats ? stats.stanced_bills_total - stats.stanced_bills_with_votes : undefined,
|
||||||
|
countLabel: "stanced bills missing votes",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "member-trends",
|
||||||
|
name: "Member Trend Scores",
|
||||||
|
description: "Calculate newsworthiness scores for legislators based on recent news coverage. Runs automatically nightly.",
|
||||||
|
fn: adminAPI.triggerMemberTrendScores,
|
||||||
|
status: "on-demand",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
key: "actions",
|
key: "actions",
|
||||||
name: "Fetch Bill Actions",
|
name: "Fetch Bill Actions",
|
||||||
|
|||||||
@@ -208,6 +208,10 @@ export interface ApiHealth {
|
|||||||
govinfo: ApiHealthResult;
|
govinfo: ApiHealthResult;
|
||||||
newsapi: ApiHealthResult;
|
newsapi: ApiHealthResult;
|
||||||
google_news: ApiHealthResult;
|
google_news: ApiHealthResult;
|
||||||
|
rep_lookup: ApiHealthResult;
|
||||||
|
redis: ApiHealthResult;
|
||||||
|
smtp: ApiHealthResult;
|
||||||
|
pytrends: ApiHealthResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AnalysisStats {
|
export interface AnalysisStats {
|
||||||
@@ -224,6 +228,13 @@ export interface AnalysisStats {
|
|||||||
bills_missing_actions: number;
|
bills_missing_actions: number;
|
||||||
unlabeled_briefs: number;
|
unlabeled_briefs: number;
|
||||||
remaining: number;
|
remaining: number;
|
||||||
|
total_members: number;
|
||||||
|
house_count: number;
|
||||||
|
senate_count: number;
|
||||||
|
members_missing_chamber: number;
|
||||||
|
total_votes: number;
|
||||||
|
stanced_bills_total: number;
|
||||||
|
stanced_bills_with_votes: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface NotificationTestResult {
|
export interface NotificationTestResult {
|
||||||
@@ -277,6 +288,12 @@ export const adminAPI = {
|
|||||||
apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data),
|
apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data),
|
||||||
triggerTrendScores: () =>
|
triggerTrendScores: () =>
|
||||||
apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data),
|
apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data),
|
||||||
|
triggerFetchNews: () =>
|
||||||
|
apiClient.post("/api/admin/trigger-fetch-news").then((r) => r.data),
|
||||||
|
triggerFetchVotes: () =>
|
||||||
|
apiClient.post("/api/admin/trigger-fetch-votes").then((r) => r.data),
|
||||||
|
triggerMemberTrendScores: () =>
|
||||||
|
apiClient.post("/api/admin/trigger-member-trend-scores").then((r) => r.data),
|
||||||
backfillSponsors: () =>
|
backfillSponsors: () =>
|
||||||
apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data),
|
apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data),
|
||||||
backfillCitations: () =>
|
backfillCitations: () =>
|
||||||
|
|||||||
Reference in New Issue
Block a user