feat: per-user notifications (ntfy + RSS), deduplicated actions, backfill task
Notifications: - New /notifications page accessible to all users (ntfy + RSS config) - ntfy now supports no-auth, Bearer token, and HTTP Basic auth (for ACL-protected self-hosted servers) - RSS enabled/disabled independently of ntfy; token auto-generated on first GET - Notification settings removed from admin-only Settings page; replaced with link card - Sidebar adds Notifications nav link for all users - notification_dispatcher.py: fan-out now marks RSS events dispatched independently Action history: - Migration 0012: deduplicates existing bill_actions rows and adds UNIQUE(bill_id, action_date, action_text) - congress_poller.py: replaces existence-check inserts with ON CONFLICT DO NOTHING (race-condition safe) - Added backfill_all_bill_actions task (no date filter) + admin endpoint POST /backfill-all-actions Authored-By: Jack Levy
This commit is contained in:
@@ -7,10 +7,11 @@ News correlation service.
|
||||
import logging
|
||||
import time
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import feedparser
|
||||
import redis
|
||||
import requests
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
@@ -22,6 +23,34 @@ NEWSAPI_BASE = "https://newsapi.org/v2"
|
||||
GOOGLE_NEWS_RSS = "https://news.google.com/rss/search"
|
||||
NEWSAPI_DAILY_LIMIT = 95 # Leave 5 as buffer
|
||||
|
||||
_NEWSAPI_REDIS_PREFIX = "newsapi:daily_calls:"
|
||||
|
||||
|
||||
def _newsapi_redis():
|
||||
return redis.from_url(settings.REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
def _newsapi_quota_ok() -> bool:
|
||||
"""Return True if we have quota remaining for today."""
|
||||
try:
|
||||
key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
|
||||
used = int(_newsapi_redis().get(key) or 0)
|
||||
return used < NEWSAPI_DAILY_LIMIT
|
||||
except Exception:
|
||||
return True # Don't block on Redis errors
|
||||
|
||||
|
||||
def _newsapi_record_call():
|
||||
try:
|
||||
r = _newsapi_redis()
|
||||
key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
|
||||
pipe = r.pipeline()
|
||||
pipe.incr(key)
|
||||
pipe.expire(key, 90000) # 25 hours — expires safely after midnight
|
||||
pipe.execute()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(2), wait=wait_exponential(min=1, max=5))
|
||||
def _newsapi_get(endpoint: str, params: dict) -> dict:
|
||||
@@ -51,6 +80,9 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
|
||||
"""Fetch articles from NewsAPI.org. Returns empty list if quota is exhausted or key not set."""
|
||||
if not settings.NEWSAPI_KEY:
|
||||
return []
|
||||
if not _newsapi_quota_ok():
|
||||
logger.warning("NewsAPI daily quota exhausted — skipping fetch")
|
||||
return []
|
||||
try:
|
||||
from_date = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
data = _newsapi_get("everything", {
|
||||
@@ -60,6 +92,7 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
|
||||
"pageSize": 10,
|
||||
"from": from_date,
|
||||
})
|
||||
_newsapi_record_call()
|
||||
articles = data.get("articles", [])
|
||||
return [
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user