feat: per-user notifications (ntfy + RSS), deduplicated actions, backfill task
Notifications: - New /notifications page accessible to all users (ntfy + RSS config) - ntfy now supports no-auth, Bearer token, and HTTP Basic auth (for ACL-protected self-hosted servers) - RSS enabled/disabled independently of ntfy; token auto-generated on first GET - Notification settings removed from admin-only Settings page; replaced with link card - Sidebar adds Notifications nav link for all users - notification_dispatcher.py: fan-out now marks RSS events dispatched independently Action history: - Migration 0012: deduplicates existing bill_actions rows and adds UNIQUE(bill_id, action_date, action_text) - congress_poller.py: replaces existence-check inserts with ON CONFLICT DO NOTHING (race-condition safe) - Added backfill_all_bill_actions task (no date filter) + admin endpoint POST /backfill-all-actions Authored-By: Jack Levy
This commit is contained in:
@@ -10,6 +10,7 @@ import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from app.database import get_sync_db
|
||||
from app.models import Bill, BillAction, Member, AppSetting
|
||||
@@ -227,30 +228,15 @@ def fetch_bill_actions(self, bill_id: str):
|
||||
break
|
||||
|
||||
for action in actions_data:
|
||||
action_date_str = action.get("actionDate")
|
||||
action_text = action.get("text", "")
|
||||
action_type = action.get("type")
|
||||
chamber = action.get("chamber")
|
||||
|
||||
# Idempotency check: skip if (bill_id, action_date, action_text) exists
|
||||
exists = (
|
||||
db.query(BillAction)
|
||||
.filter(
|
||||
BillAction.bill_id == bill_id,
|
||||
BillAction.action_date == action_date_str,
|
||||
BillAction.action_text == action_text,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if not exists:
|
||||
db.add(BillAction(
|
||||
bill_id=bill_id,
|
||||
action_date=action_date_str,
|
||||
action_text=action_text,
|
||||
action_type=action_type,
|
||||
chamber=chamber,
|
||||
))
|
||||
inserted += 1
|
||||
stmt = pg_insert(BillAction.__table__).values(
|
||||
bill_id=bill_id,
|
||||
action_date=action.get("actionDate"),
|
||||
action_text=action.get("text", ""),
|
||||
action_type=action.get("type"),
|
||||
chamber=action.get("chamber"),
|
||||
).on_conflict_do_nothing(constraint="uq_bill_actions_bill_date_text")
|
||||
result = db.execute(stmt)
|
||||
inserted += result.rowcount
|
||||
|
||||
db.commit()
|
||||
offset += 250
|
||||
@@ -297,6 +283,28 @@ def fetch_actions_for_active_bills(self):
|
||||
db.close()
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.workers.congress_poller.backfill_all_bill_actions")
|
||||
def backfill_all_bill_actions(self):
|
||||
"""One-time backfill: enqueue action fetches for every bill that has never had actions fetched."""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
bills = (
|
||||
db.query(Bill)
|
||||
.filter(Bill.actions_fetched_at.is_(None))
|
||||
.order_by(Bill.latest_action_date.desc())
|
||||
.all()
|
||||
)
|
||||
queued = 0
|
||||
for bill in bills:
|
||||
fetch_bill_actions.delay(bill.bill_id)
|
||||
queued += 1
|
||||
time.sleep(0.05) # ~20 tasks/sec — workers will self-throttle against Congress.gov
|
||||
logger.info(f"backfill_all_bill_actions: queued {queued} bills")
|
||||
return {"queued": queued}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _update_bill_if_changed(db, existing: Bill, parsed: dict) -> bool:
|
||||
"""Update bill fields if anything has changed. Returns True if updated."""
|
||||
changed = False
|
||||
|
||||
Reference in New Issue
Block a user