feat: per-user notifications (ntfy + RSS), deduplicated actions, backfill task
Notifications: - New /notifications page accessible to all users (ntfy + RSS config) - ntfy now supports no-auth, Bearer token, and HTTP Basic auth (for ACL-protected self-hosted servers) - RSS enabled/disabled independently of ntfy; token auto-generated on first GET - Notification settings removed from admin-only Settings page; replaced with link card - Sidebar adds Notifications nav link for all users - notification_dispatcher.py: fan-out now marks RSS events dispatched independently Action history: - Migration 0012: deduplicates existing bill_actions rows and adds UNIQUE(bill_id, action_date, action_text) - congress_poller.py: replaces existence-check inserts with ON CONFLICT DO NOTHING (race-condition safe) - Added backfill_all_bill_actions task (no date filter) + admin endpoint POST /backfill-all-actions Authored-By: Jack Levy
This commit is contained in:
@@ -15,6 +15,30 @@ from app.config import settings
|
||||
|
||||
BASE_URL = "https://api.congress.gov/v3"
|
||||
|
||||
_BILL_TYPE_SLUG = {
|
||||
"hr": "house-bill",
|
||||
"s": "senate-bill",
|
||||
"hjres": "house-joint-resolution",
|
||||
"sjres": "senate-joint-resolution",
|
||||
"hres": "house-resolution",
|
||||
"sres": "senate-resolution",
|
||||
"hconres": "house-concurrent-resolution",
|
||||
"sconres": "senate-concurrent-resolution",
|
||||
}
|
||||
|
||||
|
||||
def _congress_ordinal(n: int) -> str:
|
||||
if 11 <= n % 100 <= 13:
|
||||
return f"{n}th"
|
||||
suffixes = {1: "st", 2: "nd", 3: "rd"}
|
||||
return f"{n}{suffixes.get(n % 10, 'th')}"
|
||||
|
||||
|
||||
def build_bill_public_url(congress: int, bill_type: str, bill_number: int) -> str:
|
||||
"""Return the public congress.gov page URL for a bill (not the API endpoint)."""
|
||||
slug = _BILL_TYPE_SLUG.get(bill_type.lower(), bill_type.lower())
|
||||
return f"https://www.congress.gov/bill/{_congress_ordinal(congress)}-congress/{slug}/{bill_number}"
|
||||
|
||||
|
||||
def _get_current_congress() -> int:
|
||||
"""Calculate the current Congress number. 119th started Jan 3, 2025."""
|
||||
@@ -98,7 +122,7 @@ def parse_bill_from_api(data: dict, congress: int) -> dict:
|
||||
"latest_action_text": latest_action.get("text"),
|
||||
"status": latest_action.get("text", "")[:100] if latest_action.get("text") else None,
|
||||
"chamber": "House" if bill_type.startswith("h") else "Senate",
|
||||
"congress_url": data.get("url"),
|
||||
"congress_url": build_bill_public_url(congress, bill_type, bill_number),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -7,10 +7,11 @@ News correlation service.
|
||||
import logging
|
||||
import time
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import feedparser
|
||||
import redis
|
||||
import requests
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
@@ -22,6 +23,34 @@ NEWSAPI_BASE = "https://newsapi.org/v2"
|
||||
GOOGLE_NEWS_RSS = "https://news.google.com/rss/search"
|
||||
NEWSAPI_DAILY_LIMIT = 95 # Leave 5 as buffer
|
||||
|
||||
_NEWSAPI_REDIS_PREFIX = "newsapi:daily_calls:"
|
||||
|
||||
|
||||
def _newsapi_redis():
|
||||
return redis.from_url(settings.REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
def _newsapi_quota_ok() -> bool:
|
||||
"""Return True if we have quota remaining for today."""
|
||||
try:
|
||||
key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
|
||||
used = int(_newsapi_redis().get(key) or 0)
|
||||
return used < NEWSAPI_DAILY_LIMIT
|
||||
except Exception:
|
||||
return True # Don't block on Redis errors
|
||||
|
||||
|
||||
def _newsapi_record_call():
|
||||
try:
|
||||
r = _newsapi_redis()
|
||||
key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
|
||||
pipe = r.pipeline()
|
||||
pipe.incr(key)
|
||||
pipe.expire(key, 90000) # 25 hours — expires safely after midnight
|
||||
pipe.execute()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(2), wait=wait_exponential(min=1, max=5))
|
||||
def _newsapi_get(endpoint: str, params: dict) -> dict:
|
||||
@@ -51,6 +80,9 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
|
||||
"""Fetch articles from NewsAPI.org. Returns empty list if quota is exhausted or key not set."""
|
||||
if not settings.NEWSAPI_KEY:
|
||||
return []
|
||||
if not _newsapi_quota_ok():
|
||||
logger.warning("NewsAPI daily quota exhausted — skipping fetch")
|
||||
return []
|
||||
try:
|
||||
from_date = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
data = _newsapi_get("everything", {
|
||||
@@ -60,6 +92,7 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
|
||||
"pageSize": 10,
|
||||
"from": from_date,
|
||||
})
|
||||
_newsapi_record_call()
|
||||
articles = data.get("articles", [])
|
||||
return [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user