feat: roll-call votes + granular alert filter fix (v0.9.5)
Roll-call votes:
- Migration 0017: bill_votes + member_vote_positions tables
- Fetch vote XML directly from House Clerk / Senate LIS URLs
embedded in bill actions recordedVotes objects
- GET /api/bills/{id}/votes triggers background fetch on first view
- VotePanel on bill detail: yea/nay bar, result badge, followed
member positions with Sen./Rep. title, party badge, and state
Alert filter fix:
- _should_dispatch returns True when alert_filters is None so users
who haven't saved filters still receive all notifications
Authored-By: Jack Levy
This commit is contained in:
Jack Levy
@@ -12,6 +12,7 @@ from app.schemas.schemas import (
|
||||
BillDetailSchema,
|
||||
BillSchema,
|
||||
BillActionSchema,
|
||||
BillVoteSchema,
|
||||
NewsArticleSchema,
|
||||
PaginatedResponse,
|
||||
TrendScoreSchema,
|
||||
@@ -202,6 +203,32 @@ async def get_bill_trend(bill_id: str, days: int = Query(30, ge=7, le=365), db:
|
||||
return result.scalars().all()
|
||||
|
||||
|
||||
@router.get("/{bill_id}/votes", response_model=list[BillVoteSchema])
|
||||
async def get_bill_votes_endpoint(bill_id: str, db: AsyncSession = Depends(get_db)):
|
||||
from app.models.vote import BillVote
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
result = await db.execute(
|
||||
select(BillVote)
|
||||
.where(BillVote.bill_id == bill_id)
|
||||
.options(selectinload(BillVote.positions))
|
||||
.order_by(desc(BillVote.vote_date))
|
||||
)
|
||||
votes = result.scalars().unique().all()
|
||||
|
||||
# Trigger background fetch if no votes are stored yet
|
||||
if not votes:
|
||||
bill = await db.get(Bill, bill_id)
|
||||
if bill:
|
||||
try:
|
||||
from app.workers.vote_fetcher import fetch_bill_votes
|
||||
fetch_bill_votes.delay(bill_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return votes
|
||||
|
||||
|
||||
@router.post("/{bill_id}/draft-letter", response_model=DraftLetterResponse)
|
||||
async def generate_letter(bill_id: str, body: DraftLetterRequest, db: AsyncSession = Depends(get_db)):
|
||||
from app.models.setting import AppSetting
|
||||
|
||||
@@ -11,6 +11,7 @@ from app.models.setting import AppSetting
|
||||
from app.models.trend import TrendScore
|
||||
from app.models.committee import Committee, CommitteeBill
|
||||
from app.models.user import User
|
||||
from app.models.vote import BillVote, MemberVotePosition
|
||||
|
||||
__all__ = [
|
||||
"Bill",
|
||||
@@ -18,12 +19,14 @@ __all__ = [
|
||||
"BillDocument",
|
||||
"BillBrief",
|
||||
"BillNote",
|
||||
"BillVote",
|
||||
"Collection",
|
||||
"CollectionBill",
|
||||
"Follow",
|
||||
"Member",
|
||||
"MemberTrendScore",
|
||||
"MemberNewsArticle",
|
||||
"MemberVotePosition",
|
||||
"NewsArticle",
|
||||
"NotificationEvent",
|
||||
"AppSetting",
|
||||
|
||||
53
backend/app/models/vote.py
Normal file
53
backend/app/models/vote.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from sqlalchemy import Column, Date, DateTime, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship
|
||||
from sqlalchemy.sql import func
|
||||
|
||||
from app.database import Base
|
||||
|
||||
|
||||
class BillVote(Base):
|
||||
__tablename__ = "bill_votes"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
bill_id = Column(String, ForeignKey("bills.bill_id", ondelete="CASCADE"), nullable=False)
|
||||
congress = Column(Integer, nullable=False)
|
||||
chamber = Column(String(50), nullable=False)
|
||||
session = Column(Integer, nullable=False)
|
||||
roll_number = Column(Integer, nullable=False)
|
||||
question = Column(Text)
|
||||
description = Column(Text)
|
||||
vote_date = Column(Date)
|
||||
yeas = Column(Integer)
|
||||
nays = Column(Integer)
|
||||
not_voting = Column(Integer)
|
||||
result = Column(String(200))
|
||||
source_url = Column(String)
|
||||
fetched_at = Column(DateTime(timezone=True))
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
positions = relationship("MemberVotePosition", back_populates="vote", cascade="all, delete-orphan")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_bill_votes_bill_id", "bill_id"),
|
||||
UniqueConstraint("congress", "chamber", "session", "roll_number", name="uq_bill_votes_roll"),
|
||||
)
|
||||
|
||||
|
||||
class MemberVotePosition(Base):
|
||||
__tablename__ = "member_vote_positions"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
vote_id = Column(Integer, ForeignKey("bill_votes.id", ondelete="CASCADE"), nullable=False)
|
||||
bioguide_id = Column(String, ForeignKey("members.bioguide_id", ondelete="SET NULL"), nullable=True)
|
||||
member_name = Column(String(200))
|
||||
party = Column(String(50))
|
||||
state = Column(String(10))
|
||||
position = Column(String(50), nullable=False)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
vote = relationship("BillVote", back_populates="positions")
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_member_vote_positions_vote_id", "vote_id"),
|
||||
Index("ix_member_vote_positions_bioguide_id", "bioguide_id"),
|
||||
)
|
||||
@@ -337,3 +337,34 @@ class CollectionDetailSchema(CollectionSchema):
|
||||
class BriefShareResponse(BaseModel):
|
||||
brief: BriefSchema
|
||||
bill: BillSchema
|
||||
|
||||
|
||||
# ── Votes ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class MemberVotePositionSchema(BaseModel):
|
||||
bioguide_id: Optional[str] = None
|
||||
member_name: Optional[str] = None
|
||||
party: Optional[str] = None
|
||||
state: Optional[str] = None
|
||||
position: str
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class BillVoteSchema(BaseModel):
|
||||
id: int
|
||||
congress: int
|
||||
chamber: str
|
||||
session: int
|
||||
roll_number: int
|
||||
question: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
vote_date: Optional[date] = None
|
||||
yeas: Optional[int] = None
|
||||
nays: Optional[int] = None
|
||||
not_voting: Optional[int] = None
|
||||
result: Optional[str] = None
|
||||
source_url: Optional[str] = None
|
||||
positions: list[MemberVotePositionSchema] = []
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
@@ -90,6 +90,11 @@ def get_bill_text_versions(congress: int, bill_type: str, bill_number: int) -> d
|
||||
return _get(f"/bill/{congress}/{bill_type.lower()}/{bill_number}/text", {})
|
||||
|
||||
|
||||
def get_vote_detail(congress: int, chamber: str, session: int, roll_number: int) -> dict:
|
||||
chamber_slug = "house" if chamber.lower() == "house" else "senate"
|
||||
return _get(f"/vote/{congress}/{chamber_slug}/{session}/{roll_number}", {})
|
||||
|
||||
|
||||
def get_members(offset: int = 0, limit: int = 250, current_member: bool = True) -> dict:
|
||||
params: dict = {"offset": offset, "limit": limit}
|
||||
if current_member:
|
||||
|
||||
@@ -66,7 +66,9 @@ def _should_dispatch(event, prefs: dict, follow_mode: str = "neutral") -> bool:
|
||||
if not key:
|
||||
key = "referral" if payload.get("milestone_tier") == "referral" else "vote"
|
||||
|
||||
all_filters = prefs.get("alert_filters") or {}
|
||||
all_filters = prefs.get("alert_filters")
|
||||
if all_filters is None:
|
||||
return True # user hasn't configured filters yet — send everything
|
||||
mode_filters = all_filters.get(follow_mode) or {}
|
||||
return bool(mode_filters.get(key, _FILTER_DEFAULTS.get(key, True)))
|
||||
|
||||
|
||||
239
backend/app/workers/vote_fetcher.py
Normal file
239
backend/app/workers/vote_fetcher.py
Normal file
@@ -0,0 +1,239 @@
|
||||
"""
|
||||
Vote fetcher — fetches roll-call vote data for bills.
|
||||
|
||||
Roll-call votes are referenced in bill actions as recordedVotes objects.
|
||||
Each recordedVote contains a direct URL to the source XML:
|
||||
- House: https://clerk.house.gov/evs/{year}/roll{NNN}.xml
|
||||
- Senate: https://www.senate.gov/legislative/LIS/roll_call_votes/...
|
||||
|
||||
We fetch and parse that XML directly rather than going through a
|
||||
Congress.gov API endpoint (which doesn't expose vote detail).
|
||||
|
||||
Triggered on-demand from GET /api/bills/{bill_id}/votes when no votes
|
||||
are stored yet.
|
||||
"""
|
||||
import logging
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
import requests
|
||||
|
||||
from app.database import get_sync_db
|
||||
from app.models.bill import Bill
|
||||
from app.models.member import Member
|
||||
from app.models.vote import BillVote, MemberVotePosition
|
||||
from app.services.congress_api import get_bill_actions as _api_get_bill_actions
|
||||
from app.workers.celery_app import celery_app
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FETCH_TIMEOUT = 15
|
||||
|
||||
|
||||
def _parse_date(s) -> date | None:
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(str(s)[:10])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_xml(url: str) -> ET.Element:
|
||||
resp = requests.get(url, timeout=_FETCH_TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
return ET.fromstring(resp.content)
|
||||
|
||||
|
||||
def _parse_house_xml(root: ET.Element) -> dict:
|
||||
"""Parse House Clerk roll-call XML (clerk.house.gov/evs/...)."""
|
||||
meta = root.find("vote-metadata")
|
||||
question = (meta.findtext("vote-question") or "").strip() if meta is not None else ""
|
||||
result = (meta.findtext("vote-result") or "").strip() if meta is not None else ""
|
||||
|
||||
totals = root.find(".//totals-by-vote")
|
||||
yeas = int((totals.findtext("yea-total") or "0").strip()) if totals is not None else 0
|
||||
nays = int((totals.findtext("nay-total") or "0").strip()) if totals is not None else 0
|
||||
not_voting = int((totals.findtext("not-voting-total") or "0").strip()) if totals is not None else 0
|
||||
|
||||
members = []
|
||||
for rv in root.findall(".//recorded-vote"):
|
||||
leg = rv.find("legislator")
|
||||
if leg is None:
|
||||
continue
|
||||
members.append({
|
||||
"bioguide_id": leg.get("name-id"),
|
||||
"member_name": (leg.text or "").strip(),
|
||||
"party": leg.get("party"),
|
||||
"state": leg.get("state"),
|
||||
"position": (rv.findtext("vote") or "Not Voting").strip(),
|
||||
})
|
||||
|
||||
return {"question": question, "result": result, "yeas": yeas, "nays": nays,
|
||||
"not_voting": not_voting, "members": members}
|
||||
|
||||
|
||||
def _parse_senate_xml(root: ET.Element) -> dict:
|
||||
"""Parse Senate LIS roll-call XML (senate.gov/legislative/LIS/...)."""
|
||||
question = (root.findtext("vote_question_text") or root.findtext("question") or "").strip()
|
||||
result = (root.findtext("vote_result_text") or "").strip()
|
||||
|
||||
counts = root.find("vote_counts")
|
||||
yeas = int((counts.findtext("yeas") or "0").strip()) if counts is not None else 0
|
||||
nays = int((counts.findtext("nays") or "0").strip()) if counts is not None else 0
|
||||
not_voting = int((counts.findtext("absent") or "0").strip()) if counts is not None else 0
|
||||
|
||||
members = []
|
||||
for m in root.findall(".//member"):
|
||||
first = (m.findtext("first_name") or "").strip()
|
||||
last = (m.findtext("last_name") or "").strip()
|
||||
members.append({
|
||||
"bioguide_id": (m.findtext("bioguide_id") or "").strip() or None,
|
||||
"member_name": f"{first} {last}".strip(),
|
||||
"party": m.findtext("party"),
|
||||
"state": m.findtext("state"),
|
||||
"position": (m.findtext("vote_cast") or "Not Voting").strip(),
|
||||
})
|
||||
|
||||
return {"question": question, "result": result, "yeas": yeas, "nays": nays,
|
||||
"not_voting": not_voting, "members": members}
|
||||
|
||||
|
||||
def _parse_vote_xml(url: str, chamber: str) -> dict:
|
||||
root = _fetch_xml(url)
|
||||
if chamber.lower() == "house":
|
||||
return _parse_house_xml(root)
|
||||
return _parse_senate_xml(root)
|
||||
|
||||
|
||||
def _collect_recorded_votes(congress: int, bill_type: str, bill_number: int) -> list[dict]:
|
||||
"""Page through all bill actions and collect unique recordedVotes entries."""
|
||||
seen: set[tuple] = set()
|
||||
recorded: list[dict] = []
|
||||
offset = 0
|
||||
|
||||
while True:
|
||||
data = _api_get_bill_actions(congress, bill_type, bill_number, offset=offset)
|
||||
actions = data.get("actions", [])
|
||||
pagination = data.get("pagination", {})
|
||||
|
||||
for action in actions:
|
||||
for rv in action.get("recordedVotes", []):
|
||||
chamber = rv.get("chamber", "")
|
||||
session = int(rv.get("sessionNumber") or rv.get("session") or 1)
|
||||
roll_number = rv.get("rollNumber")
|
||||
if not roll_number:
|
||||
continue
|
||||
roll_number = int(roll_number)
|
||||
key = (chamber, session, roll_number)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
recorded.append({
|
||||
"chamber": chamber,
|
||||
"session": session,
|
||||
"roll_number": roll_number,
|
||||
"date": action.get("actionDate"),
|
||||
"url": rv.get("url"),
|
||||
})
|
||||
|
||||
total = pagination.get("count", 0)
|
||||
offset += len(actions)
|
||||
if offset >= total or not actions:
|
||||
break
|
||||
|
||||
return recorded
|
||||
|
||||
|
||||
@celery_app.task(bind=True, name="app.workers.vote_fetcher.fetch_bill_votes")
|
||||
def fetch_bill_votes(self, bill_id: str) -> dict:
|
||||
"""Fetch and store roll-call votes for a single bill."""
|
||||
db = get_sync_db()
|
||||
try:
|
||||
bill = db.get(Bill, bill_id)
|
||||
if not bill:
|
||||
return {"error": f"Bill {bill_id} not found"}
|
||||
|
||||
recorded = _collect_recorded_votes(bill.congress_number, bill.bill_type, bill.bill_number)
|
||||
|
||||
if not recorded:
|
||||
logger.info(f"fetch_bill_votes({bill_id}): no recorded votes in actions")
|
||||
return {"bill_id": bill_id, "stored": 0, "skipped": 0}
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
stored = 0
|
||||
skipped = 0
|
||||
|
||||
# Cache known bioguide IDs to avoid N+1 member lookups
|
||||
known_bioguides: set[str] = {
|
||||
row[0] for row in db.query(Member.bioguide_id).all()
|
||||
}
|
||||
|
||||
for rv in recorded:
|
||||
chamber = rv["chamber"]
|
||||
session = rv["session"]
|
||||
roll_number = rv["roll_number"]
|
||||
source_url = rv.get("url")
|
||||
|
||||
existing = (
|
||||
db.query(BillVote)
|
||||
.filter_by(
|
||||
congress=bill.congress_number,
|
||||
chamber=chamber,
|
||||
session=session,
|
||||
roll_number=roll_number,
|
||||
)
|
||||
.first()
|
||||
)
|
||||
if existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if not source_url:
|
||||
logger.warning(f"No URL for {chamber} roll {roll_number} — skipping")
|
||||
continue
|
||||
|
||||
try:
|
||||
parsed = _parse_vote_xml(source_url, chamber)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Could not parse vote XML {source_url}: {exc}")
|
||||
continue
|
||||
|
||||
bill_vote = BillVote(
|
||||
bill_id=bill_id,
|
||||
congress=bill.congress_number,
|
||||
chamber=chamber,
|
||||
session=session,
|
||||
roll_number=roll_number,
|
||||
question=parsed["question"],
|
||||
description=None,
|
||||
vote_date=_parse_date(rv.get("date")),
|
||||
yeas=parsed["yeas"],
|
||||
nays=parsed["nays"],
|
||||
not_voting=parsed["not_voting"],
|
||||
result=parsed["result"],
|
||||
source_url=source_url,
|
||||
fetched_at=now,
|
||||
)
|
||||
db.add(bill_vote)
|
||||
db.flush()
|
||||
|
||||
for pos in parsed["members"]:
|
||||
bioguide_id = pos.get("bioguide_id")
|
||||
if bioguide_id and bioguide_id not in known_bioguides:
|
||||
bioguide_id = None
|
||||
db.add(MemberVotePosition(
|
||||
vote_id=bill_vote.id,
|
||||
bioguide_id=bioguide_id,
|
||||
member_name=pos.get("member_name"),
|
||||
party=pos.get("party"),
|
||||
state=pos.get("state"),
|
||||
position=pos.get("position") or "Not Voting",
|
||||
))
|
||||
|
||||
db.commit()
|
||||
stored += 1
|
||||
|
||||
logger.info(f"fetch_bill_votes({bill_id}): {stored} stored, {skipped} skipped")
|
||||
return {"bill_id": bill_id, "stored": stored, "skipped": skipped}
|
||||
finally:
|
||||
db.close()
|
||||
Reference in New Issue
Block a user