feat: per-user notifications (ntfy + RSS), deduplicated actions, backfill task
Notifications: - New /notifications page accessible to all users (ntfy + RSS config) - ntfy now supports no-auth, Bearer token, and HTTP Basic auth (for ACL-protected self-hosted servers) - RSS enabled/disabled independently of ntfy; token auto-generated on first GET - Notification settings removed from admin-only Settings page; replaced with link card - Sidebar adds Notifications nav link for all users - notification_dispatcher.py: fan-out now marks RSS events dispatched independently Action history: - Migration 0012: deduplicates existing bill_actions rows and adds UNIQUE(bill_id, action_date, action_text) - congress_poller.py: replaces existence-check inserts with ON CONFLICT DO NOTHING (race-condition safe) - Added backfill_all_bill_actions task (no date filter) + admin endpoint POST /backfill-all-actions Authored-By: Jack Levy
This commit is contained in:
@@ -130,6 +130,10 @@ async def get_stats(
|
||||
WHERE bb.id IS NULL AND bd.raw_text IS NOT NULL
|
||||
""")
|
||||
)).scalar()
|
||||
# Bills that have never had their action history fetched
|
||||
bills_missing_actions = (await db.execute(
|
||||
text("SELECT COUNT(*) FROM bills WHERE actions_fetched_at IS NULL")
|
||||
)).scalar()
|
||||
return {
|
||||
"total_bills": total_bills,
|
||||
"docs_fetched": docs_fetched,
|
||||
@@ -141,6 +145,7 @@ async def get_stats(
|
||||
"pending_llm": pending_llm,
|
||||
"bills_missing_sponsor": bills_missing_sponsor,
|
||||
"bills_missing_metadata": bills_missing_metadata,
|
||||
"bills_missing_actions": bills_missing_actions,
|
||||
"remaining": total_bills - total_briefs,
|
||||
}
|
||||
|
||||
@@ -183,6 +188,14 @@ async def trigger_fetch_actions(current_user: User = Depends(get_current_admin))
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/backfill-all-actions")
|
||||
async def backfill_all_actions(current_user: User = Depends(get_current_admin)):
|
||||
"""Queue action fetches for every bill that has never had actions fetched."""
|
||||
from app.workers.congress_poller import backfill_all_bill_actions
|
||||
task = backfill_all_bill_actions.delay()
|
||||
return {"task_id": task.id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/backfill-metadata")
|
||||
async def backfill_metadata(current_user: User = Depends(get_current_admin)):
|
||||
"""Fill in null introduced_date, congress_url, chamber for existing bills."""
|
||||
|
||||
@@ -69,12 +69,11 @@ async def get_member(bioguide_id: str, db: AsyncSession = Depends(get_db)):
|
||||
if not member:
|
||||
raise HTTPException(status_code=404, detail="Member not found")
|
||||
|
||||
# Kick off member interest scoring on first view (non-blocking)
|
||||
# Kick off member interest on first view — single combined task avoids duplicate API calls
|
||||
if member.detail_fetched is None:
|
||||
try:
|
||||
from app.workers.member_interest import fetch_member_news, calculate_member_trend_score
|
||||
fetch_member_news.delay(bioguide_id)
|
||||
calculate_member_trend_score.delay(bioguide_id)
|
||||
from app.workers.member_interest import sync_member_interest
|
||||
sync_member_interest.delay(bioguide_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
138
backend/app/api/notifications.py
Normal file
138
backend/app/api/notifications.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""
|
||||
Notifications API — user notification settings and per-user RSS feed.
|
||||
"""
|
||||
import secrets
|
||||
from xml.etree.ElementTree import Element, SubElement, tostring
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.dependencies import get_current_user
|
||||
from app.database import get_db
|
||||
from app.models.notification import NotificationEvent
|
||||
from app.models.user import User
|
||||
from app.schemas.schemas import NotificationSettingsResponse, NotificationSettingsUpdate
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_EVENT_LABELS = {
|
||||
"new_document": "New Bill Text",
|
||||
"new_amendment": "Amendment Filed",
|
||||
"bill_updated": "Bill Updated",
|
||||
}
|
||||
|
||||
|
||||
def _prefs_to_response(prefs: dict, rss_token: str | None) -> NotificationSettingsResponse:
|
||||
return NotificationSettingsResponse(
|
||||
ntfy_topic_url=prefs.get("ntfy_topic_url", ""),
|
||||
ntfy_auth_method=prefs.get("ntfy_auth_method", "none"),
|
||||
ntfy_token=prefs.get("ntfy_token", ""),
|
||||
ntfy_username=prefs.get("ntfy_username", ""),
|
||||
ntfy_password=prefs.get("ntfy_password", ""),
|
||||
ntfy_enabled=prefs.get("ntfy_enabled", False),
|
||||
rss_enabled=prefs.get("rss_enabled", False),
|
||||
rss_token=rss_token,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/settings", response_model=NotificationSettingsResponse)
|
||||
async def get_notification_settings(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
user = await db.get(User, current_user.id)
|
||||
# Auto-generate RSS token on first visit so the feed URL is always available
|
||||
if not user.rss_token:
|
||||
user.rss_token = secrets.token_urlsafe(32)
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return _prefs_to_response(user.notification_prefs or {}, user.rss_token)
|
||||
|
||||
|
||||
@router.put("/settings", response_model=NotificationSettingsResponse)
|
||||
async def update_notification_settings(
|
||||
body: NotificationSettingsUpdate,
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
user = await db.get(User, current_user.id)
|
||||
prefs = dict(user.notification_prefs or {})
|
||||
|
||||
if body.ntfy_topic_url is not None:
|
||||
prefs["ntfy_topic_url"] = body.ntfy_topic_url.strip()
|
||||
if body.ntfy_auth_method is not None:
|
||||
prefs["ntfy_auth_method"] = body.ntfy_auth_method
|
||||
if body.ntfy_token is not None:
|
||||
prefs["ntfy_token"] = body.ntfy_token.strip()
|
||||
if body.ntfy_username is not None:
|
||||
prefs["ntfy_username"] = body.ntfy_username.strip()
|
||||
if body.ntfy_password is not None:
|
||||
prefs["ntfy_password"] = body.ntfy_password.strip()
|
||||
if body.ntfy_enabled is not None:
|
||||
prefs["ntfy_enabled"] = body.ntfy_enabled
|
||||
if body.rss_enabled is not None:
|
||||
prefs["rss_enabled"] = body.rss_enabled
|
||||
|
||||
user.notification_prefs = prefs
|
||||
|
||||
if not user.rss_token:
|
||||
user.rss_token = secrets.token_urlsafe(32)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return _prefs_to_response(user.notification_prefs or {}, user.rss_token)
|
||||
|
||||
|
||||
@router.post("/settings/rss-reset", response_model=NotificationSettingsResponse)
|
||||
async def reset_rss_token(
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
"""Regenerate the RSS token, invalidating the old feed URL."""
|
||||
user = await db.get(User, current_user.id)
|
||||
user.rss_token = secrets.token_urlsafe(32)
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return _prefs_to_response(user.notification_prefs or {}, user.rss_token)
|
||||
|
||||
|
||||
@router.get("/feed/{rss_token}.xml", include_in_schema=False)
|
||||
async def rss_feed(rss_token: str, db: AsyncSession = Depends(get_db)):
|
||||
"""Public tokenized RSS feed — no auth required."""
|
||||
result = await db.execute(select(User).where(User.rss_token == rss_token))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="Feed not found")
|
||||
|
||||
events_result = await db.execute(
|
||||
select(NotificationEvent)
|
||||
.where(NotificationEvent.user_id == user.id)
|
||||
.order_by(NotificationEvent.created_at.desc())
|
||||
.limit(50)
|
||||
)
|
||||
events = events_result.scalars().all()
|
||||
return Response(content=_build_rss(events), media_type="application/rss+xml")
|
||||
|
||||
|
||||
def _build_rss(events: list) -> bytes:
|
||||
rss = Element("rss", version="2.0")
|
||||
channel = SubElement(rss, "channel")
|
||||
SubElement(channel, "title").text = "PocketVeto — Bill Alerts"
|
||||
SubElement(channel, "description").text = "Updates on your followed bills"
|
||||
SubElement(channel, "language").text = "en-us"
|
||||
|
||||
for event in events:
|
||||
payload = event.payload or {}
|
||||
item = SubElement(channel, "item")
|
||||
label = _EVENT_LABELS.get(event.event_type, "Update")
|
||||
bill_label = payload.get("bill_label", event.bill_id.upper())
|
||||
SubElement(item, "title").text = f"{label}: {bill_label} — {payload.get('bill_title', '')}"
|
||||
SubElement(item, "description").text = payload.get("brief_summary", "")
|
||||
if payload.get("bill_url"):
|
||||
SubElement(item, "link").text = payload["bill_url"]
|
||||
SubElement(item, "pubDate").text = event.created_at.strftime("%a, %d %b %Y %H:%M:%S +0000")
|
||||
SubElement(item, "guid").text = str(event.id)
|
||||
|
||||
return tostring(rss, encoding="unicode").encode("utf-8")
|
||||
@@ -1,7 +1,7 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
|
||||
from app.api import bills, members, follows, dashboard, search, settings, admin, health, auth
|
||||
from app.api import bills, members, follows, dashboard, search, settings, admin, health, auth, notifications
|
||||
from app.config import settings as config
|
||||
|
||||
app = FastAPI(
|
||||
@@ -27,3 +27,4 @@ app.include_router(search.router, prefix="/api/search", tags=["search"])
|
||||
app.include_router(settings.router, prefix="/api/settings", tags=["settings"])
|
||||
app.include_router(admin.router, prefix="/api/admin", tags=["admin"])
|
||||
app.include_router(health.router, prefix="/api/health", tags=["health"])
|
||||
app.include_router(notifications.router, prefix="/api/notifications", tags=["notifications"])
|
||||
|
||||
@@ -4,6 +4,7 @@ from app.models.follow import Follow
|
||||
from app.models.member import Member
|
||||
from app.models.member_interest import MemberTrendScore, MemberNewsArticle
|
||||
from app.models.news import NewsArticle
|
||||
from app.models.notification import NotificationEvent
|
||||
from app.models.setting import AppSetting
|
||||
from app.models.trend import TrendScore
|
||||
from app.models.committee import Committee, CommitteeBill
|
||||
@@ -19,6 +20,7 @@ __all__ = [
|
||||
"MemberTrendScore",
|
||||
"MemberNewsArticle",
|
||||
"NewsArticle",
|
||||
"NotificationEvent",
|
||||
"AppSetting",
|
||||
"TrendScore",
|
||||
"Committee",
|
||||
|
||||
27
backend/app/models/notification.py
Normal file
27
backend/app/models/notification.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, String
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class NotificationEvent(Base):
|
||||
__tablename__ = "notification_events"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False)
|
||||
bill_id = Column(String, ForeignKey("bills.bill_id", ondelete="CASCADE"), nullable=False)
|
||||
# new_document | new_amendment | bill_updated
|
||||
event_type = Column(String(50), nullable=False)
|
||||
# {bill_title, bill_label, brief_summary, bill_url}
|
||||
payload = Column(JSONB)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
dispatched_at = Column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
user = relationship("User", back_populates="notification_events")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_notification_events_user_id", "user_id"),
|
||||
Index("ix_notification_events_dispatched_at", "dispatched_at"),
|
||||
)
|
||||
@@ -14,6 +14,8 @@ class User(Base):
|
||||
hashed_password = Column(String, nullable=False)
|
||||
is_admin = Column(Boolean, nullable=False, default=False)
|
||||
notification_prefs = Column(JSONB, nullable=False, default=dict)
|
||||
rss_token = Column(String, unique=True, nullable=True, index=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
follows = relationship("Follow", back_populates="user", cascade="all, delete-orphan")
|
||||
notification_events = relationship("NotificationEvent", back_populates="user", cascade="all, delete-orphan")
|
||||
|
||||
@@ -8,8 +8,12 @@ from pydantic import BaseModel
|
||||
|
||||
class NotificationSettingsResponse(BaseModel):
|
||||
ntfy_topic_url: str = ""
|
||||
ntfy_auth_method: str = "none" # none | token | basic
|
||||
ntfy_token: str = ""
|
||||
ntfy_username: str = ""
|
||||
ntfy_password: str = ""
|
||||
ntfy_enabled: bool = False
|
||||
rss_enabled: bool = False
|
||||
rss_token: Optional[str] = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
@@ -17,8 +21,12 @@ class NotificationSettingsResponse(BaseModel):
|
||||
|
||||
class NotificationSettingsUpdate(BaseModel):
|
||||
ntfy_topic_url: Optional[str] = None
|
||||
ntfy_auth_method: Optional[str] = None
|
||||
ntfy_token: Optional[str] = None
|
||||
ntfy_username: Optional[str] = None
|
||||
ntfy_password: Optional[str] = None
|
||||
ntfy_enabled: Optional[bool] = None
|
||||
rss_enabled: Optional[bool] = None
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
@@ -15,6 +15,30 @@ from app.config import settings
|
||||
|
||||
BASE_URL = "https://api.congress.gov/v3"
|
||||
|
||||
_BILL_TYPE_SLUG = {
|
||||
"hr": "house-bill",
|
||||
"s": "senate-bill",
|
||||
"hjres": "house-joint-resolution",
|
||||
"sjres": "senate-joint-resolution",
|
||||
"hres": "house-resolution",
|
||||
"sres": "senate-resolution",
|
||||
"hconres": "house-concurrent-resolution",
|
||||
"sconres": "senate-concurrent-resolution",
|
||||
}
|
||||
|
||||
|
||||
def _congress_ordinal(n: int) -> str:
|
||||
if 11 <= n % 100 <= 13:
|
||||
return f"{n}th"
|
||||
suffixes = {1: "st", 2: "nd", 3: "rd"}
|
||||
return f"{n}{suffixes.get(n % 10, 'th')}"
|
||||
|
||||
|
||||
def build_bill_public_url(congress: int, bill_type: str, bill_number: int) -> str:
|
||||
"""Return the public congress.gov page URL for a bill (not the API endpoint)."""
|
||||
slug = _BILL_TYPE_SLUG.get(bill_type.lower(), bill_type.lower())
|
||||
return f"https://www.congress.gov/bill/{_congress_ordinal(congress)}-congress/{slug}/{bill_number}"
|
||||
|
||||
|
||||
def _get_current_congress() -> int:
|
||||
"""Calculate the current Congress number. 119th started Jan 3, 2025."""
|
||||
@@ -98,7 +122,7 @@ def parse_bill_from_api(data: dict, congress: int) -> dict:
|
||||
"latest_action_text": latest_action.get("text"),
|
||||
"status": latest_action.get("text", "")[:100] if latest_action.get("text") else None,
|
||||
"chamber": "House" if bill_type.startswith("h") else "Senate",
|
||||
"congress_url": data.get("url"),
|
||||
"congress_url": build_bill_public_url(congress, bill_type, bill_number),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -7,10 +7,11 @@ News correlation service.
|
||||
import logging
|
||||
import time
|
||||
import urllib.parse
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import feedparser
|
||||
import redis
|
||||
import requests
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
@@ -22,6 +23,34 @@ NEWSAPI_BASE = "https://newsapi.org/v2"
|
||||
GOOGLE_NEWS_RSS = "https://news.google.com/rss/search"
|
||||
NEWSAPI_DAILY_LIMIT = 95 # Leave 5 as buffer
|
||||
|
||||
_NEWSAPI_REDIS_PREFIX = "newsapi:daily_calls:"
|
||||
|
||||
|
||||
def _newsapi_redis():
|
||||
return redis.from_url(settings.REDIS_URL, decode_responses=True)
|
||||
|
||||
|
||||
def _newsapi_quota_ok() -> bool:
|
||||
"""Return True if we have quota remaining for today."""
|
||||
try:
|
||||
key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
|
||||
used = int(_newsapi_redis().get(key) or 0)
|
||||
return used < NEWSAPI_DAILY_LIMIT
|
||||
except Exception:
|
||||
return True # Don't block on Redis errors
|
||||
|
||||
|
||||
def _newsapi_record_call():
|
||||
try:
|
||||
r = _newsapi_redis()
|
||||
key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
|
||||
pipe = r.pipeline()
|
||||
pipe.incr(key)
|
||||
pipe.expire(key, 90000) # 25 hours — expires safely after midnight
|
||||
pipe.execute()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@retry(stop=stop_after_attempt(2), wait=wait_exponential(min=1, max=5))
|
||||
def _newsapi_get(endpoint: str, params: dict) -> dict:
|
||||
@@ -51,6 +80,9 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
|
||||
"""Fetch articles from NewsAPI.org. Returns empty list if quota is exhausted or key not set."""
|
||||
if not settings.NEWSAPI_KEY:
|
||||
return []
|
||||
if not _newsapi_quota_ok():
|
||||
logger.warning("NewsAPI daily quota exhausted — skipping fetch")
|
||||
return []
|
||||
try:
|
||||
from_date = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
data = _newsapi_get("everything", {
|
||||
@@ -60,6 +92,7 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
|
||||
"pageSize": 10,
|
||||
"from": from_date,
|
||||
})
|
||||
_newsapi_record_call()
|
||||
articles = data.get("articles", [])
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -10,6 +10,7 @@ import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from app.database import get_sync_db
|
||||
from app.models import Bill, BillAction, Member, AppSetting
|
||||
@@ -227,30 +228,15 @@ def fetch_bill_actions(self, bill_id: str):
|
||||
break
|
||||
|
||||
for action in actions_data:
|
||||
action_date_str = action.get("actionDate")
|
||||
action_text = action.get("text", "")
|
||||
action_type = action.get("type")
|
||||
chamber = action.get("chamber")
|
||||
|
||||
# Idempotency check: skip if (bill_id, action_date, action_text) exists
|
||||
exists = (
|
||||
db.query(BillAction)
|
||||
.filter(
|
||||
BillAction.bill_id == bill_id,
|
||||
BillAction.action_date == action_date_str,
|
||||
BillAction.action_text == action_text,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not exists:
|
||||
db.add(BillAction(
|
||||
bill_id=bill_id,
|
||||
action_date=action_date_str,
|
||||
action_text=action_text,
|
||||
action_type=action_type,
|
||||
chamber=chamber,
|
||||
))
|
||||
inserted += 1
|
||||
stmt = pg_insert(BillAction.__table__).values(
|
||||
bill_id=bill_id,
|
||||
action_date=action.get("actionDate"),
|
||||
action_text=action.get("text", ""),
|
||||
action_type=action.get("type"),
|
||||
chamber=action.get("chamber"),
|
||||
).on_conflict_do_nothing(constraint="uq_bill_actions_bill_date_text")
|
||||
result = db.execute(stmt)
|
||||
inserted += result.rowcount
|
||||
|
||||
db.commit()
|
||||
offset += 250
|
||||
@@ -297,6 +283,28 @@ def fetch_actions_for_active_bills(self):
|
||||
db.close()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.workers.congress_poller.backfill_all_bill_actions")
|
||||
def backfill_all_bill_actions(self):
|
||||
"""One-time backfill: enqueue action fetches for every bill that has never had actions fetched."""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
bills = (
|
||||
db.query(Bill)
|
||||
.filter(Bill.actions_fetched_at.is_(None))
|
||||
.order_by(Bill.latest_action_date.desc())
|
||||
.all()
|
||||
)
|
||||
queued = 0
|
||||
for bill in bills:
|
||||
fetch_bill_actions.delay(bill.bill_id)
|
||||
queued += 1
|
||||
time.sleep(0.05) # ~20 tasks/sec — workers will self-throttle against Congress.gov
|
||||
logger.info(f"backfill_all_bill_actions: queued {queued} bills")
|
||||
return {"queued": queued}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _update_bill_if_changed(db, existing: Bill, parsed: dict) -> bool:
|
||||
"""Update bill fields if anything has changed. Returns True if updated."""
|
||||
changed = False
|
||||
|
||||
@@ -26,6 +26,81 @@ def _parse_pub_at(raw: str | None) -> datetime | None:
|
||||
return None
|
||||
|
||||
|
||||
@celery_app.task(bind=True, max_retries=2, name="app.workers.member_interest.sync_member_interest")
|
||||
def sync_member_interest(self, bioguide_id: str):
|
||||
"""
|
||||
Fetch news and score a member in a single API pass.
|
||||
Called on first profile view — avoids the 2x NewsAPI + GNews calls that
|
||||
result from queuing fetch_member_news and calculate_member_trend_score separately.
|
||||
"""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
member = db.get(Member, bioguide_id)
|
||||
if not member or not member.first_name or not member.last_name:
|
||||
return {"status": "skipped"}
|
||||
|
||||
query = news_service.build_member_query(
|
||||
first_name=member.first_name,
|
||||
last_name=member.last_name,
|
||||
chamber=member.chamber,
|
||||
)
|
||||
|
||||
# Single fetch — results reused for both article storage and scoring
|
||||
newsapi_articles = news_service.fetch_newsapi_articles(query, days=30)
|
||||
gnews_articles = news_service.fetch_gnews_articles(query, days=30)
|
||||
all_articles = newsapi_articles + gnews_articles
|
||||
|
||||
saved = 0
|
||||
for article in all_articles:
|
||||
url = article.get("url")
|
||||
if not url:
|
||||
continue
|
||||
existing = (
|
||||
db.query(MemberNewsArticle)
|
||||
.filter_by(member_id=bioguide_id, url=url)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
continue
|
||||
db.add(MemberNewsArticle(
|
||||
member_id=bioguide_id,
|
||||
source=article.get("source", "")[:200],
|
||||
headline=article.get("headline", ""),
|
||||
url=url,
|
||||
published_at=_parse_pub_at(article.get("published_at")),
|
||||
relevance_score=1.0,
|
||||
))
|
||||
saved += 1
|
||||
|
||||
# Score using counts already in hand — no second API round-trip
|
||||
today = date.today()
|
||||
if not db.query(MemberTrendScore).filter_by(member_id=bioguide_id, score_date=today).first():
|
||||
keywords = trends_service.keywords_for_member(member.first_name, member.last_name)
|
||||
gtrends_score = trends_service.get_trends_score(keywords)
|
||||
composite = calculate_composite_score(
|
||||
len(newsapi_articles), len(gnews_articles), gtrends_score
|
||||
)
|
||||
db.add(MemberTrendScore(
|
||||
member_id=bioguide_id,
|
||||
score_date=today,
|
||||
newsapi_count=len(newsapi_articles),
|
||||
gnews_count=len(gnews_articles),
|
||||
gtrends_score=gtrends_score,
|
||||
composite_score=composite,
|
||||
))
|
||||
|
||||
db.commit()
|
||||
logger.info(f"Synced member interest for {bioguide_id}: {saved} articles saved")
|
||||
return {"status": "ok", "saved": saved}
|
||||
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
logger.error(f"Member interest sync failed for {bioguide_id}: {exc}")
|
||||
raise self.retry(exc=exc, countdown=300)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, max_retries=2, name="app.workers.member_interest.fetch_member_news")
|
||||
def fetch_member_news(self, bioguide_id: str):
|
||||
"""Fetch and store recent news articles for a specific member."""
|
||||
|
||||
115
backend/app/workers/notification_dispatcher.py
Normal file
115
backend/app/workers/notification_dispatcher.py
Normal file
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
Notification dispatcher — sends pending notification events via ntfy.
|
||||
|
||||
RSS is pull-based so no dispatch is needed for it; events are simply
|
||||
marked dispatched once ntfy is sent (or immediately if the user has no
|
||||
ntfy configured but has an RSS token, so the feed can clean up old items).
|
||||
|
||||
Runs every 5 minutes on Celery Beat.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
from app.database import get_sync_db
|
||||
from app.models.notification import NotificationEvent
|
||||
from app.models.user import User
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NTFY_TIMEOUT = 10
|
||||
|
||||
_EVENT_TITLES = {
|
||||
"new_document": "New Bill Text Published",
|
||||
"new_amendment": "Amendment Filed",
|
||||
"bill_updated": "Bill Updated",
|
||||
}
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.workers.notification_dispatcher.dispatch_notifications")
|
||||
def dispatch_notifications(self):
|
||||
"""Fan out pending notification events to ntfy and mark dispatched."""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
pending = (
|
||||
db.query(NotificationEvent)
|
||||
.filter(NotificationEvent.dispatched_at.is_(None))
|
||||
.order_by(NotificationEvent.created_at)
|
||||
.limit(200)
|
||||
.all()
|
||||
)
|
||||
|
||||
sent = 0
|
||||
failed = 0
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for event in pending:
|
||||
user = db.get(User, event.user_id)
|
||||
if not user:
|
||||
event.dispatched_at = now
|
||||
db.commit()
|
||||
continue
|
||||
|
||||
prefs = user.notification_prefs or {}
|
||||
ntfy_url = prefs.get("ntfy_topic_url", "").strip()
|
||||
ntfy_auth_method = prefs.get("ntfy_auth_method", "none")
|
||||
ntfy_token = prefs.get("ntfy_token", "").strip()
|
||||
ntfy_username = prefs.get("ntfy_username", "").strip()
|
||||
ntfy_password = prefs.get("ntfy_password", "").strip()
|
||||
ntfy_enabled = prefs.get("ntfy_enabled", False)
|
||||
rss_enabled = prefs.get("rss_enabled", False)
|
||||
|
||||
if ntfy_enabled and ntfy_url:
|
||||
try:
|
||||
_send_ntfy(event, ntfy_url, ntfy_auth_method, ntfy_token, ntfy_username, ntfy_password)
|
||||
sent += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"ntfy dispatch failed for event {event.id}: {e}")
|
||||
failed += 1
|
||||
|
||||
# Mark dispatched once handled by at least one enabled channel.
|
||||
# RSS is pull-based — no action needed beyond creating the event record.
|
||||
if (ntfy_enabled and ntfy_url) or rss_enabled:
|
||||
event.dispatched_at = now
|
||||
db.commit()
|
||||
|
||||
logger.info(f"dispatch_notifications: {sent} sent, {failed} failed, {len(pending)} pending")
|
||||
return {"sent": sent, "failed": failed, "total": len(pending)}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _send_ntfy(
|
||||
event: NotificationEvent,
|
||||
topic_url: str,
|
||||
auth_method: str = "none",
|
||||
token: str = "",
|
||||
username: str = "",
|
||||
password: str = "",
|
||||
) -> None:
|
||||
import base64
|
||||
payload = event.payload or {}
|
||||
bill_label = payload.get("bill_label", event.bill_id.upper())
|
||||
bill_title = payload.get("bill_title", "")
|
||||
message = f"{bill_label}: {bill_title}"
|
||||
if payload.get("brief_summary"):
|
||||
message += f"\n\n{payload['brief_summary'][:280]}"
|
||||
|
||||
headers = {
|
||||
"Title": _EVENT_TITLES.get(event.event_type, "Bill Update"),
|
||||
"Priority": "default",
|
||||
"Tags": "scroll",
|
||||
}
|
||||
if payload.get("bill_url"):
|
||||
headers["Click"] = payload["bill_url"]
|
||||
|
||||
if auth_method == "token" and token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
elif auth_method == "basic" and username:
|
||||
creds = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
headers["Authorization"] = f"Basic {creds}"
|
||||
|
||||
resp = requests.post(topic_url, data=message.encode("utf-8"), headers=headers, timeout=NTFY_TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
Reference in New Issue
Block a user