Nine efficiency improvements across the data pipeline:
1. NewsAPI OR batching (news_service.py + news_fetcher.py)
- Combine up to 4 bills per NewsAPI call using OR query syntax
- NEWSAPI_BATCH_SIZE=4 means ~4× effective daily quota (100→400 bill-fetches)
- fetch_news_for_bill_batch task; fetch_news_for_active_bills queues batches
2. Google News RSS cache (news_service.py)
- 2-hour Redis cache shared between news_fetcher and trend_scorer
- Eliminates duplicate RSS hits when both workers run against same bill
- clear_gnews_cache() admin helper + admin endpoint
3. pytrends keyword batching (trends_service.py + trend_scorer.py)
- Compare up to 5 bills per pytrends call instead of 1
- get_trends_scores_batch() returns scores in original order
- Reduces pytrends calls by ~5× and associated rate-limit risk
4. GovInfo ETags (govinfo_api.py + document_fetcher.py)
- If-None-Match conditional GET; DocumentUnchangedError on HTTP 304
- ETags stored in Redis (30-day TTL) keyed by MD5(url)
- document_fetcher catches DocumentUnchangedError → {"status": "unchanged"}
5. Anthropic prompt caching (llm_service.py)
- cache_control: {type: ephemeral} on system messages in AnthropicProvider
- Caches the ~700-token system prompt server-side; ~50% cost reduction on
repeated calls within the 5-minute cache window
6. Async sponsor fetch (congress_poller.py)
- New fetch_sponsor_for_bill Celery task replaces blocking get_bill_detail()
inline in poll loop
- Bills saved immediately with sponsor_id=None; sponsor linked async
- Removes 0.25s sleep per new bill from poll hot path
7. Skip doc fetch for procedural actions (congress_poller.py)
- _DOC_PRODUCING_CATEGORIES = {vote, committee_report, presidential, ...}
- fetch_bill_documents only enqueued when action is likely to produce
new GovInfo text (saves ~60–70% of unnecessary document fetch attempts)
8. Adaptive poll frequency (congress_poller.py)
- _is_congress_off_hours(): weekends + before 9AM / after 9PM EST
- Skips poll if off-hours AND last poll < 1 hour ago
- Prevents wasteful polling when Congress is not in session
9. Admin panel additions (admin.py + settings/page.tsx + api.ts)
- GET /api/admin/newsapi-quota → remaining calls today
- POST /api/admin/clear-gnews-cache → flush RSS cache
- Settings page shows NewsAPI quota remaining (amber if < 10)
- "Clear Google News Cache" button in Manual Controls
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
301 lines
10 KiB
TypeScript
301 lines
10 KiB
TypeScript
import axios from "axios";
|
|
import type {
|
|
Bill,
|
|
BillAction,
|
|
BillDetail,
|
|
BillVote,
|
|
BriefSchema,
|
|
Collection,
|
|
CollectionDetail,
|
|
DashboardData,
|
|
Follow,
|
|
Member,
|
|
MemberTrendScore,
|
|
MemberNewsArticle,
|
|
NewsArticle,
|
|
NotificationEvent,
|
|
NotificationSettings,
|
|
PaginatedResponse,
|
|
SettingsData,
|
|
TrendScore,
|
|
} from "./types";
|
|
|
|
const apiClient = axios.create({
|
|
baseURL: process.env.NEXT_PUBLIC_API_URL || "",
|
|
timeout: 30000,
|
|
});
|
|
|
|
// Attach JWT from localStorage on every request
|
|
apiClient.interceptors.request.use((config) => {
|
|
if (typeof window !== "undefined") {
|
|
try {
|
|
const stored = localStorage.getItem("pocketveto-auth");
|
|
if (stored) {
|
|
const { state } = JSON.parse(stored);
|
|
if (state?.token) {
|
|
config.headers.Authorization = `Bearer ${state.token}`;
|
|
}
|
|
}
|
|
} catch {
|
|
// ignore parse errors
|
|
}
|
|
}
|
|
return config;
|
|
});
|
|
|
|
interface AuthUser {
|
|
id: number;
|
|
email: string;
|
|
is_admin: boolean;
|
|
notification_prefs: Record<string, unknown>;
|
|
created_at: string;
|
|
}
|
|
|
|
interface TokenResponse {
|
|
access_token: string;
|
|
token_type: string;
|
|
user: AuthUser;
|
|
}
|
|
|
|
// Auth
|
|
export const authAPI = {
|
|
register: (email: string, password: string) =>
|
|
apiClient.post<TokenResponse>("/api/auth/register", { email, password }).then((r) => r.data),
|
|
login: (email: string, password: string) =>
|
|
apiClient.post<TokenResponse>("/api/auth/login", { email, password }).then((r) => r.data),
|
|
me: () =>
|
|
apiClient.get<AuthUser>("/api/auth/me").then((r) => r.data),
|
|
};
|
|
|
|
// Bills
|
|
export const billsAPI = {
|
|
list: (params?: Record<string, unknown>) =>
|
|
apiClient.get<PaginatedResponse<Bill>>("/api/bills", { params }).then((r) => r.data),
|
|
get: (id: string) =>
|
|
apiClient.get<BillDetail>(`/api/bills/${id}`).then((r) => r.data),
|
|
getActions: (id: string) =>
|
|
apiClient.get<BillAction[]>(`/api/bills/${id}/actions`).then((r) => r.data),
|
|
getNews: (id: string) =>
|
|
apiClient.get<NewsArticle[]>(`/api/bills/${id}/news`).then((r) => r.data),
|
|
getTrend: (id: string, days?: number) =>
|
|
apiClient.get<TrendScore[]>(`/api/bills/${id}/trend`, { params: { days } }).then((r) => r.data),
|
|
getVotes: (id: string) =>
|
|
apiClient.get<BillVote[]>(`/api/bills/${id}/votes`).then((r) => r.data),
|
|
generateDraft: (id: string, body: {
|
|
stance: string;
|
|
recipient: string;
|
|
tone: string;
|
|
selected_points: string[];
|
|
include_citations: boolean;
|
|
zip_code?: string;
|
|
rep_name?: string;
|
|
}) =>
|
|
apiClient.post<{ draft: string }>(`/api/bills/${id}/draft-letter`, body).then((r) => r.data),
|
|
};
|
|
|
|
// Collections
|
|
export const collectionsAPI = {
|
|
list: () =>
|
|
apiClient.get<Collection[]>("/api/collections").then((r) => r.data),
|
|
create: (name: string, is_public: boolean) =>
|
|
apiClient.post<Collection>("/api/collections", { name, is_public }).then((r) => r.data),
|
|
get: (id: number) =>
|
|
apiClient.get<CollectionDetail>(`/api/collections/${id}`).then((r) => r.data),
|
|
update: (id: number, data: { name?: string; is_public?: boolean }) =>
|
|
apiClient.patch<Collection>(`/api/collections/${id}`, data).then((r) => r.data),
|
|
delete: (id: number) => apiClient.delete(`/api/collections/${id}`),
|
|
addBill: (id: number, bill_id: string) =>
|
|
apiClient.post(`/api/collections/${id}/bills/${bill_id}`).then((r) => r.data),
|
|
removeBill: (id: number, bill_id: string) =>
|
|
apiClient.delete(`/api/collections/${id}/bills/${bill_id}`),
|
|
getByShareToken: (token: string) =>
|
|
apiClient.get<CollectionDetail>(`/api/collections/share/${token}`).then((r) => r.data),
|
|
};
|
|
|
|
// Share (public)
|
|
export const shareAPI = {
|
|
getBrief: (token: string) =>
|
|
apiClient.get<{ brief: BriefSchema; bill: Bill }>(`/api/share/brief/${token}`).then((r) => r.data),
|
|
getCollection: (token: string) =>
|
|
apiClient.get<CollectionDetail>(`/api/share/collection/${token}`).then((r) => r.data),
|
|
};
|
|
|
|
// Notes
|
|
export const notesAPI = {
|
|
get: (billId: string) =>
|
|
apiClient.get<import("./types").BillNote>(`/api/notes/${billId}`).then((r) => r.data),
|
|
upsert: (billId: string, content: string, pinned: boolean) =>
|
|
apiClient.put<import("./types").BillNote>(`/api/notes/${billId}`, { content, pinned }).then((r) => r.data),
|
|
delete: (billId: string) =>
|
|
apiClient.delete(`/api/notes/${billId}`),
|
|
};
|
|
|
|
// Members
|
|
export const membersAPI = {
|
|
list: (params?: Record<string, unknown>) =>
|
|
apiClient.get<PaginatedResponse<Member>>("/api/members", { params }).then((r) => r.data),
|
|
get: (id: string) =>
|
|
apiClient.get<Member>(`/api/members/${id}`).then((r) => r.data),
|
|
byZip: (zip: string) =>
|
|
apiClient.get<Member[]>(`/api/members/by-zip/${zip}`).then((r) => r.data),
|
|
getBills: (id: string, params?: Record<string, unknown>) =>
|
|
apiClient.get<PaginatedResponse<Bill>>(`/api/members/${id}/bills`, { params }).then((r) => r.data),
|
|
getTrend: (id: string, days?: number) =>
|
|
apiClient.get<MemberTrendScore[]>(`/api/members/${id}/trend`, { params: { days } }).then((r) => r.data),
|
|
getNews: (id: string) =>
|
|
apiClient.get<MemberNewsArticle[]>(`/api/members/${id}/news`).then((r) => r.data),
|
|
};
|
|
|
|
// Follows
|
|
export const followsAPI = {
|
|
list: () =>
|
|
apiClient.get<Follow[]>("/api/follows").then((r) => r.data),
|
|
add: (follow_type: string, follow_value: string) =>
|
|
apiClient.post<Follow>("/api/follows", { follow_type, follow_value }).then((r) => r.data),
|
|
remove: (id: number) =>
|
|
apiClient.delete(`/api/follows/${id}`),
|
|
updateMode: (id: number, mode: string) =>
|
|
apiClient.patch<Follow>(`/api/follows/${id}/mode`, { follow_mode: mode }).then((r) => r.data),
|
|
};
|
|
|
|
// Dashboard
|
|
export const dashboardAPI = {
|
|
get: () =>
|
|
apiClient.get<DashboardData>("/api/dashboard").then((r) => r.data),
|
|
};
|
|
|
|
// Search
|
|
export const searchAPI = {
|
|
search: (q: string) =>
|
|
apiClient.get<{ bills: Bill[]; members: Member[] }>("/api/search", { params: { q } }).then((r) => r.data),
|
|
};
|
|
|
|
export interface LLMModel {
|
|
id: string;
|
|
name: string;
|
|
}
|
|
|
|
// Settings
|
|
export const settingsAPI = {
|
|
get: () =>
|
|
apiClient.get<SettingsData>("/api/settings").then((r) => r.data),
|
|
update: (key: string, value: string) =>
|
|
apiClient.put("/api/settings", { key, value }).then((r) => r.data),
|
|
testLLM: () =>
|
|
apiClient.post("/api/settings/test-llm").then((r) => r.data),
|
|
listModels: (provider: string) =>
|
|
apiClient.get<{ models: LLMModel[]; error?: string }>("/api/settings/llm-models", { params: { provider } }).then((r) => r.data),
|
|
};
|
|
|
|
export interface AdminUser {
|
|
id: number;
|
|
email: string;
|
|
is_admin: boolean;
|
|
follow_count: number;
|
|
created_at: string;
|
|
}
|
|
|
|
export interface ApiHealthResult {
|
|
status: "ok" | "error" | "skipped";
|
|
detail: string;
|
|
latency_ms?: number;
|
|
}
|
|
|
|
export interface ApiHealth {
|
|
congress_gov: ApiHealthResult;
|
|
govinfo: ApiHealthResult;
|
|
newsapi: ApiHealthResult;
|
|
google_news: ApiHealthResult;
|
|
}
|
|
|
|
export interface AnalysisStats {
|
|
total_bills: number;
|
|
docs_fetched: number;
|
|
briefs_generated: number;
|
|
full_briefs: number;
|
|
amendment_briefs: number;
|
|
uncited_briefs: number;
|
|
no_text_bills: number;
|
|
pending_llm: number;
|
|
bills_missing_sponsor: number;
|
|
bills_missing_metadata: number;
|
|
bills_missing_actions: number;
|
|
unlabeled_briefs: number;
|
|
remaining: number;
|
|
}
|
|
|
|
export interface NotificationTestResult {
|
|
status: "ok" | "error";
|
|
detail: string;
|
|
event_count?: number;
|
|
}
|
|
|
|
// Notifications
|
|
export const notificationsAPI = {
|
|
getSettings: () =>
|
|
apiClient.get<NotificationSettings>("/api/notifications/settings").then((r) => r.data),
|
|
updateSettings: (data: Partial<NotificationSettings>) =>
|
|
apiClient.put<NotificationSettings>("/api/notifications/settings", data).then((r) => r.data),
|
|
resetRssToken: () =>
|
|
apiClient.post<NotificationSettings>("/api/notifications/settings/rss-reset").then((r) => r.data),
|
|
testNtfy: (data: {
|
|
ntfy_topic_url: string;
|
|
ntfy_auth_method: string;
|
|
ntfy_token: string;
|
|
ntfy_username: string;
|
|
ntfy_password: string;
|
|
}) =>
|
|
apiClient.post<NotificationTestResult>("/api/notifications/test/ntfy", data).then((r) => r.data),
|
|
testRss: () =>
|
|
apiClient.post<NotificationTestResult>("/api/notifications/test/rss").then((r) => r.data),
|
|
testFollowMode: (mode: string, event_type: string) =>
|
|
apiClient.post<NotificationTestResult>("/api/notifications/test/follow-mode", { mode, event_type }).then((r) => r.data),
|
|
getHistory: () =>
|
|
apiClient.get<NotificationEvent[]>("/api/notifications/history").then((r) => r.data),
|
|
};
|
|
|
|
// Admin
|
|
export const adminAPI = {
|
|
// Stats
|
|
getStats: () =>
|
|
apiClient.get<AnalysisStats>("/api/admin/stats").then((r) => r.data),
|
|
// Users
|
|
listUsers: () =>
|
|
apiClient.get<AdminUser[]>("/api/admin/users").then((r) => r.data),
|
|
deleteUser: (id: number) =>
|
|
apiClient.delete(`/api/admin/users/${id}`),
|
|
toggleAdmin: (id: number) =>
|
|
apiClient.patch<AdminUser>(`/api/admin/users/${id}/toggle-admin`).then((r) => r.data),
|
|
// Tasks
|
|
triggerPoll: () =>
|
|
apiClient.post("/api/admin/trigger-poll").then((r) => r.data),
|
|
triggerMemberSync: () =>
|
|
apiClient.post("/api/admin/trigger-member-sync").then((r) => r.data),
|
|
triggerTrendScores: () =>
|
|
apiClient.post("/api/admin/trigger-trend-scores").then((r) => r.data),
|
|
backfillSponsors: () =>
|
|
apiClient.post("/api/admin/backfill-sponsors").then((r) => r.data),
|
|
backfillCitations: () =>
|
|
apiClient.post("/api/admin/backfill-citations").then((r) => r.data),
|
|
triggerFetchActions: () =>
|
|
apiClient.post("/api/admin/trigger-fetch-actions").then((r) => r.data),
|
|
backfillAllActions: () =>
|
|
apiClient.post("/api/admin/backfill-all-actions").then((r) => r.data),
|
|
backfillMetadata: () =>
|
|
apiClient.post("/api/admin/backfill-metadata").then((r) => r.data),
|
|
backfillLabels: () =>
|
|
apiClient.post("/api/admin/backfill-labels").then((r) => r.data),
|
|
resumeAnalysis: () =>
|
|
apiClient.post("/api/admin/resume-analysis").then((r) => r.data),
|
|
triggerWeeklyDigest: () =>
|
|
apiClient.post("/api/admin/trigger-weekly-digest").then((r) => r.data),
|
|
getApiHealth: () =>
|
|
apiClient.get<ApiHealth>("/api/admin/api-health").then((r) => r.data),
|
|
getTaskStatus: (taskId: string) =>
|
|
apiClient.get(`/api/admin/task-status/${taskId}`).then((r) => r.data),
|
|
getNewsApiQuota: () =>
|
|
apiClient.get<{ remaining: number; limit: number }>("/api/admin/newsapi-quota").then((r) => r.data),
|
|
clearGnewsCache: () =>
|
|
apiClient.post<{ cleared: number }>("/api/admin/clear-gnews-cache").then((r) => r.data),
|
|
};
|