feat(interest): add public interest tracking for members of Congress
Adds Google Trends, NewsAPI, and Google News RSS scoring for members,
mirroring the existing bill interest pipeline. Member profiles now show
a Public Interest chart (with signal breakdown) and a Related News panel.
Key changes:
- New member_trend_scores + member_news_articles tables (migration 0008)
- fetch_gnews_articles() added to news_service for unlimited RSS article storage
- Bill news fetcher now combines NewsAPI + Google News RSS (more coverage)
- New member_interest Celery worker with scheduled news + trend tasks
- GET /members/{id}/trend and /news API endpoints
- TrendChart redesigned with signal breakdown badges and bar+line combo chart
- NewsPanel accepts generic article shape (bills and members)
Co-Authored-By: Jack Levy
This commit is contained in:
54
backend/alembic/versions/0008_add_member_interest_tables.py
Normal file
54
backend/alembic/versions/0008_add_member_interest_tables.py
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
"""add member trend scores and news articles tables
|
||||||
|
|
||||||
|
Revision ID: 0008
|
||||||
|
Revises: 0007
|
||||||
|
Create Date: 2026-03-01
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0008"
|
||||||
|
down_revision = "0007"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.create_table(
|
||||||
|
"member_trend_scores",
|
||||||
|
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("member_id", sa.String(), nullable=False),
|
||||||
|
sa.Column("score_date", sa.Date(), nullable=False),
|
||||||
|
sa.Column("newsapi_count", sa.Integer(), nullable=True, default=0),
|
||||||
|
sa.Column("gnews_count", sa.Integer(), nullable=True, default=0),
|
||||||
|
sa.Column("gtrends_score", sa.Float(), nullable=True, default=0.0),
|
||||||
|
sa.Column("composite_score", sa.Float(), nullable=True, default=0.0),
|
||||||
|
sa.ForeignKeyConstraint(["member_id"], ["members.bioguide_id"], ondelete="CASCADE"),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint("member_id", "score_date", name="uq_member_trend_scores_member_date"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_member_trend_scores_member_id", "member_trend_scores", ["member_id"])
|
||||||
|
op.create_index("ix_member_trend_scores_score_date", "member_trend_scores", ["score_date"])
|
||||||
|
op.create_index("ix_member_trend_scores_composite", "member_trend_scores", ["composite_score"])
|
||||||
|
|
||||||
|
op.create_table(
|
||||||
|
"member_news_articles",
|
||||||
|
sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
|
||||||
|
sa.Column("member_id", sa.String(), nullable=False),
|
||||||
|
sa.Column("source", sa.String(200), nullable=True),
|
||||||
|
sa.Column("headline", sa.Text(), nullable=True),
|
||||||
|
sa.Column("url", sa.String(), nullable=True),
|
||||||
|
sa.Column("published_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("relevance_score", sa.Float(), nullable=True, default=0.0),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
||||||
|
sa.ForeignKeyConstraint(["member_id"], ["members.bioguide_id"], ondelete="CASCADE"),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint("member_id", "url", name="uq_member_news_member_url"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_member_news_articles_member_id", "member_news_articles", ["member_id"])
|
||||||
|
op.create_index("ix_member_news_articles_published_at", "member_news_articles", ["published_at"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_table("member_news_articles")
|
||||||
|
op.drop_table("member_trend_scores")
|
||||||
@@ -8,8 +8,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from app.database import get_db
|
from app.database import get_db
|
||||||
from app.models import Bill, Member
|
from app.models import Bill, Member, MemberTrendScore, MemberNewsArticle
|
||||||
from app.schemas.schemas import BillSchema, MemberSchema, PaginatedResponse
|
from app.schemas.schemas import (
|
||||||
|
BillSchema, MemberSchema, MemberTrendScoreSchema,
|
||||||
|
MemberNewsArticleSchema, PaginatedResponse,
|
||||||
|
)
|
||||||
from app.services import congress_api
|
from app.services import congress_api
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -66,6 +69,15 @@ async def get_member(bioguide_id: str, db: AsyncSession = Depends(get_db)):
|
|||||||
if not member:
|
if not member:
|
||||||
raise HTTPException(status_code=404, detail="Member not found")
|
raise HTTPException(status_code=404, detail="Member not found")
|
||||||
|
|
||||||
|
# Kick off member interest scoring on first view (non-blocking)
|
||||||
|
if member.detail_fetched is None:
|
||||||
|
try:
|
||||||
|
from app.workers.member_interest import fetch_member_news, calculate_member_trend_score
|
||||||
|
fetch_member_news.delay(bioguide_id)
|
||||||
|
calculate_member_trend_score.delay(bioguide_id)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
# Lazy-enrich with detail data from Congress.gov on first view
|
# Lazy-enrich with detail data from Congress.gov on first view
|
||||||
if member.detail_fetched is None:
|
if member.detail_fetched is None:
|
||||||
try:
|
try:
|
||||||
@@ -80,7 +92,47 @@ async def get_member(bioguide_id: str, db: AsyncSession = Depends(get_db)):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"Could not enrich member detail for {bioguide_id}: {e}")
|
logger.warning(f"Could not enrich member detail for {bioguide_id}: {e}")
|
||||||
|
|
||||||
return member
|
# Attach latest trend score
|
||||||
|
result_schema = MemberSchema.model_validate(member)
|
||||||
|
latest_trend = (
|
||||||
|
await db.execute(
|
||||||
|
select(MemberTrendScore)
|
||||||
|
.where(MemberTrendScore.member_id == bioguide_id)
|
||||||
|
.order_by(desc(MemberTrendScore.score_date))
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
trend = latest_trend.scalar_one_or_none()
|
||||||
|
if trend:
|
||||||
|
result_schema.latest_trend = MemberTrendScoreSchema.model_validate(trend)
|
||||||
|
return result_schema
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{bioguide_id}/trend", response_model=list[MemberTrendScoreSchema])
|
||||||
|
async def get_member_trend(
|
||||||
|
bioguide_id: str,
|
||||||
|
days: int = Query(30, ge=7, le=365),
|
||||||
|
db: AsyncSession = Depends(get_db),
|
||||||
|
):
|
||||||
|
from datetime import date, timedelta
|
||||||
|
cutoff = date.today() - timedelta(days=days)
|
||||||
|
result = await db.execute(
|
||||||
|
select(MemberTrendScore)
|
||||||
|
.where(MemberTrendScore.member_id == bioguide_id, MemberTrendScore.score_date >= cutoff)
|
||||||
|
.order_by(MemberTrendScore.score_date)
|
||||||
|
)
|
||||||
|
return result.scalars().all()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{bioguide_id}/news", response_model=list[MemberNewsArticleSchema])
|
||||||
|
async def get_member_news(bioguide_id: str, db: AsyncSession = Depends(get_db)):
|
||||||
|
result = await db.execute(
|
||||||
|
select(MemberNewsArticle)
|
||||||
|
.where(MemberNewsArticle.member_id == bioguide_id)
|
||||||
|
.order_by(desc(MemberNewsArticle.published_at))
|
||||||
|
.limit(20)
|
||||||
|
)
|
||||||
|
return result.scalars().all()
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{bioguide_id}/bills", response_model=PaginatedResponse[BillSchema])
|
@router.get("/{bioguide_id}/bills", response_model=PaginatedResponse[BillSchema])
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ from app.models.bill import Bill, BillAction, BillDocument
|
|||||||
from app.models.brief import BillBrief
|
from app.models.brief import BillBrief
|
||||||
from app.models.follow import Follow
|
from app.models.follow import Follow
|
||||||
from app.models.member import Member
|
from app.models.member import Member
|
||||||
|
from app.models.member_interest import MemberTrendScore, MemberNewsArticle
|
||||||
from app.models.news import NewsArticle
|
from app.models.news import NewsArticle
|
||||||
from app.models.setting import AppSetting
|
from app.models.setting import AppSetting
|
||||||
from app.models.trend import TrendScore
|
from app.models.trend import TrendScore
|
||||||
@@ -15,6 +16,8 @@ __all__ = [
|
|||||||
"BillBrief",
|
"BillBrief",
|
||||||
"Follow",
|
"Follow",
|
||||||
"Member",
|
"Member",
|
||||||
|
"MemberTrendScore",
|
||||||
|
"MemberNewsArticle",
|
||||||
"NewsArticle",
|
"NewsArticle",
|
||||||
"AppSetting",
|
"AppSetting",
|
||||||
"TrendScore",
|
"TrendScore",
|
||||||
|
|||||||
@@ -31,3 +31,11 @@ class Member(Base):
|
|||||||
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
bills = relationship("Bill", back_populates="sponsor", foreign_keys="Bill.sponsor_id")
|
bills = relationship("Bill", back_populates="sponsor", foreign_keys="Bill.sponsor_id")
|
||||||
|
trend_scores = relationship(
|
||||||
|
"MemberTrendScore", back_populates="member",
|
||||||
|
order_by="desc(MemberTrendScore.score_date)", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
news_articles = relationship(
|
||||||
|
"MemberNewsArticle", back_populates="member",
|
||||||
|
order_by="desc(MemberNewsArticle.published_at)", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
|||||||
47
backend/app/models/member_interest.py
Normal file
47
backend/app/models/member_interest.py
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
from sqlalchemy import Column, Integer, String, Date, Float, Text, DateTime, ForeignKey, Index, UniqueConstraint
|
||||||
|
from sqlalchemy.orm import relationship
|
||||||
|
from sqlalchemy.sql import func
|
||||||
|
|
||||||
|
from app.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
class MemberTrendScore(Base):
|
||||||
|
__tablename__ = "member_trend_scores"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
member_id = Column(String, ForeignKey("members.bioguide_id", ondelete="CASCADE"), nullable=False)
|
||||||
|
score_date = Column(Date, nullable=False)
|
||||||
|
newsapi_count = Column(Integer, default=0)
|
||||||
|
gnews_count = Column(Integer, default=0)
|
||||||
|
gtrends_score = Column(Float, default=0.0)
|
||||||
|
composite_score = Column(Float, default=0.0)
|
||||||
|
|
||||||
|
member = relationship("Member", back_populates="trend_scores")
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("member_id", "score_date", name="uq_member_trend_scores_member_date"),
|
||||||
|
Index("ix_member_trend_scores_member_id", "member_id"),
|
||||||
|
Index("ix_member_trend_scores_score_date", "score_date"),
|
||||||
|
Index("ix_member_trend_scores_composite", "composite_score"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class MemberNewsArticle(Base):
|
||||||
|
__tablename__ = "member_news_articles"
|
||||||
|
|
||||||
|
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||||
|
member_id = Column(String, ForeignKey("members.bioguide_id", ondelete="CASCADE"), nullable=False)
|
||||||
|
source = Column(String(200))
|
||||||
|
headline = Column(Text)
|
||||||
|
url = Column(String)
|
||||||
|
published_at = Column(DateTime(timezone=True))
|
||||||
|
relevance_score = Column(Float, default=0.0)
|
||||||
|
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
member = relationship("Member", back_populates="news_articles")
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("member_id", "url", name="uq_member_news_member_url"),
|
||||||
|
Index("ix_member_news_articles_member_id", "member_id"),
|
||||||
|
Index("ix_member_news_articles_published_at", "published_at"),
|
||||||
|
)
|
||||||
@@ -35,6 +35,7 @@ class MemberSchema(BaseModel):
|
|||||||
leadership_json: Optional[list[Any]] = None
|
leadership_json: Optional[list[Any]] = None
|
||||||
sponsored_count: Optional[int] = None
|
sponsored_count: Optional[int] = None
|
||||||
cosponsored_count: Optional[int] = None
|
cosponsored_count: Optional[int] = None
|
||||||
|
latest_trend: Optional["MemberTrendScoreSchema"] = None
|
||||||
|
|
||||||
model_config = {"from_attributes": True}
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
@@ -94,6 +95,27 @@ class TrendScoreSchema(BaseModel):
|
|||||||
model_config = {"from_attributes": True}
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class MemberTrendScoreSchema(BaseModel):
|
||||||
|
score_date: date
|
||||||
|
newsapi_count: int
|
||||||
|
gnews_count: int
|
||||||
|
gtrends_score: float
|
||||||
|
composite_score: float
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class MemberNewsArticleSchema(BaseModel):
|
||||||
|
id: int
|
||||||
|
source: Optional[str] = None
|
||||||
|
headline: Optional[str] = None
|
||||||
|
url: Optional[str] = None
|
||||||
|
published_at: Optional[datetime] = None
|
||||||
|
relevance_score: Optional[float] = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
# ── Bill ──────────────────────────────────────────────────────────────────────
|
# ── Bill ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
class BillSchema(BaseModel):
|
class BillSchema(BaseModel):
|
||||||
|
|||||||
@@ -87,3 +87,52 @@ def fetch_gnews_count(query: str, days: int = 30) -> int:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Google News RSS fetch failed: {e}")
|
logger.error(f"Google News RSS fetch failed: {e}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_gnews_articles(query: str, days: int = 30) -> list[dict]:
|
||||||
|
"""Fetch articles from Google News RSS. No rate limit — unlimited source."""
|
||||||
|
import time as time_mod
|
||||||
|
try:
|
||||||
|
encoded = urllib.parse.quote(f"{query} when:{days}d")
|
||||||
|
url = f"{GOOGLE_NEWS_RSS}?q={encoded}&hl=en-US&gl=US&ceid=US:en"
|
||||||
|
time.sleep(1) # Polite delay
|
||||||
|
feed = feedparser.parse(url)
|
||||||
|
articles = []
|
||||||
|
for entry in feed.entries[:20]:
|
||||||
|
pub_at = None
|
||||||
|
if entry.get("published_parsed"):
|
||||||
|
try:
|
||||||
|
pub_at = datetime.fromtimestamp(
|
||||||
|
time_mod.mktime(entry.published_parsed), tz=timezone.utc
|
||||||
|
).isoformat()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
source = ""
|
||||||
|
if hasattr(entry, "source") and isinstance(entry.source, dict):
|
||||||
|
source = entry.source.get("title", "")
|
||||||
|
elif entry.get("tags"):
|
||||||
|
source = entry.tags[0].get("term", "") if entry.tags else ""
|
||||||
|
articles.append({
|
||||||
|
"source": source or "Google News",
|
||||||
|
"headline": entry.get("title", ""),
|
||||||
|
"url": entry.get("link", ""),
|
||||||
|
"published_at": pub_at,
|
||||||
|
})
|
||||||
|
return [a for a in articles if a["url"] and a["headline"]]
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Google News RSS article fetch failed: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def build_member_query(first_name: str, last_name: str, chamber: Optional[str] = None) -> str:
|
||||||
|
"""Build a news search query for a member of Congress."""
|
||||||
|
full_name = f"{first_name} {last_name}".strip()
|
||||||
|
title = ""
|
||||||
|
if chamber:
|
||||||
|
if "senate" in chamber.lower():
|
||||||
|
title = "Senator"
|
||||||
|
else:
|
||||||
|
title = "Rep."
|
||||||
|
if title:
|
||||||
|
return f'"{full_name}" OR "{title} {last_name}"'
|
||||||
|
return f'"{full_name}"'
|
||||||
|
|||||||
@@ -50,6 +50,14 @@ def get_trends_score(keywords: list[str]) -> float:
|
|||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def keywords_for_member(first_name: str, last_name: str) -> list[str]:
|
||||||
|
"""Extract meaningful search keywords for a member of Congress."""
|
||||||
|
full_name = f"{first_name} {last_name}".strip()
|
||||||
|
if not full_name:
|
||||||
|
return []
|
||||||
|
return [full_name]
|
||||||
|
|
||||||
|
|
||||||
def keywords_for_bill(title: str, short_title: str, topic_tags: list[str]) -> list[str]:
|
def keywords_for_bill(title: str, short_title: str, topic_tags: list[str]) -> list[str]:
|
||||||
"""Extract meaningful search keywords for a bill."""
|
"""Extract meaningful search keywords for a bill."""
|
||||||
keywords = []
|
keywords = []
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ celery_app = Celery(
|
|||||||
"app.workers.llm_processor",
|
"app.workers.llm_processor",
|
||||||
"app.workers.news_fetcher",
|
"app.workers.news_fetcher",
|
||||||
"app.workers.trend_scorer",
|
"app.workers.trend_scorer",
|
||||||
|
"app.workers.member_interest",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -35,6 +36,7 @@ celery_app.conf.update(
|
|||||||
"app.workers.llm_processor.*": {"queue": "llm"},
|
"app.workers.llm_processor.*": {"queue": "llm"},
|
||||||
"app.workers.news_fetcher.*": {"queue": "news"},
|
"app.workers.news_fetcher.*": {"queue": "news"},
|
||||||
"app.workers.trend_scorer.*": {"queue": "news"},
|
"app.workers.trend_scorer.*": {"queue": "news"},
|
||||||
|
"app.workers.member_interest.*": {"queue": "news"},
|
||||||
},
|
},
|
||||||
task_queues=[
|
task_queues=[
|
||||||
Queue("polling"),
|
Queue("polling"),
|
||||||
@@ -58,5 +60,13 @@ celery_app.conf.update(
|
|||||||
"task": "app.workers.trend_scorer.calculate_all_trend_scores",
|
"task": "app.workers.trend_scorer.calculate_all_trend_scores",
|
||||||
"schedule": crontab(hour=2, minute=0),
|
"schedule": crontab(hour=2, minute=0),
|
||||||
},
|
},
|
||||||
|
"fetch-news-active-members": {
|
||||||
|
"task": "app.workers.member_interest.fetch_news_for_active_members",
|
||||||
|
"schedule": crontab(hour="*/12", minute=30),
|
||||||
|
},
|
||||||
|
"calculate-member-trend-scores": {
|
||||||
|
"task": "app.workers.member_interest.calculate_all_member_trend_scores",
|
||||||
|
"schedule": crontab(hour=3, minute=0),
|
||||||
|
},
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|||||||
177
backend/app/workers/member_interest.py
Normal file
177
backend/app/workers/member_interest.py
Normal file
@@ -0,0 +1,177 @@
|
|||||||
|
"""
|
||||||
|
Member interest worker — tracks public interest in members of Congress.
|
||||||
|
|
||||||
|
Fetches news articles and calculates trend scores for members using the
|
||||||
|
same composite scoring model as bills (NewsAPI + Google News RSS + pytrends).
|
||||||
|
Runs on a schedule and can also be triggered per-member.
|
||||||
|
"""
|
||||||
|
import logging
|
||||||
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from app.database import get_sync_db
|
||||||
|
from app.models import Member, MemberNewsArticle, MemberTrendScore
|
||||||
|
from app.services import news_service, trends_service
|
||||||
|
from app.workers.celery_app import celery_app
|
||||||
|
from app.workers.trend_scorer import calculate_composite_score
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_pub_at(raw: str | None) -> datetime | None:
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(bind=True, max_retries=2, name="app.workers.member_interest.fetch_member_news")
|
||||||
|
def fetch_member_news(self, bioguide_id: str):
|
||||||
|
"""Fetch and store recent news articles for a specific member."""
|
||||||
|
db = get_sync_db()
|
||||||
|
try:
|
||||||
|
member = db.get(Member, bioguide_id)
|
||||||
|
if not member or not member.first_name or not member.last_name:
|
||||||
|
return {"status": "skipped"}
|
||||||
|
|
||||||
|
query = news_service.build_member_query(
|
||||||
|
first_name=member.first_name,
|
||||||
|
last_name=member.last_name,
|
||||||
|
chamber=member.chamber,
|
||||||
|
)
|
||||||
|
|
||||||
|
newsapi_articles = news_service.fetch_newsapi_articles(query, days=30)
|
||||||
|
gnews_articles = news_service.fetch_gnews_articles(query, days=30)
|
||||||
|
all_articles = newsapi_articles + gnews_articles
|
||||||
|
|
||||||
|
saved = 0
|
||||||
|
for article in all_articles:
|
||||||
|
url = article.get("url")
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
existing = (
|
||||||
|
db.query(MemberNewsArticle)
|
||||||
|
.filter_by(member_id=bioguide_id, url=url)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
continue
|
||||||
|
db.add(MemberNewsArticle(
|
||||||
|
member_id=bioguide_id,
|
||||||
|
source=article.get("source", "")[:200],
|
||||||
|
headline=article.get("headline", ""),
|
||||||
|
url=url,
|
||||||
|
published_at=_parse_pub_at(article.get("published_at")),
|
||||||
|
relevance_score=1.0,
|
||||||
|
))
|
||||||
|
saved += 1
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
logger.info(f"Saved {saved} news articles for member {bioguide_id}")
|
||||||
|
return {"status": "ok", "saved": saved}
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
db.rollback()
|
||||||
|
logger.error(f"Member news fetch failed for {bioguide_id}: {exc}")
|
||||||
|
raise self.retry(exc=exc, countdown=300)
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(bind=True, name="app.workers.member_interest.calculate_member_trend_score")
|
||||||
|
def calculate_member_trend_score(self, bioguide_id: str):
|
||||||
|
"""Calculate and store today's public interest score for a member."""
|
||||||
|
db = get_sync_db()
|
||||||
|
try:
|
||||||
|
member = db.get(Member, bioguide_id)
|
||||||
|
if not member or not member.first_name or not member.last_name:
|
||||||
|
return {"status": "skipped"}
|
||||||
|
|
||||||
|
today = date.today()
|
||||||
|
existing = (
|
||||||
|
db.query(MemberTrendScore)
|
||||||
|
.filter_by(member_id=bioguide_id, score_date=today)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
if existing:
|
||||||
|
return {"status": "already_scored"}
|
||||||
|
|
||||||
|
query = news_service.build_member_query(
|
||||||
|
first_name=member.first_name,
|
||||||
|
last_name=member.last_name,
|
||||||
|
chamber=member.chamber,
|
||||||
|
)
|
||||||
|
keywords = trends_service.keywords_for_member(member.first_name, member.last_name)
|
||||||
|
|
||||||
|
newsapi_articles = news_service.fetch_newsapi_articles(query, days=30)
|
||||||
|
newsapi_count = len(newsapi_articles)
|
||||||
|
gnews_count = news_service.fetch_gnews_count(query, days=30)
|
||||||
|
gtrends_score = trends_service.get_trends_score(keywords)
|
||||||
|
|
||||||
|
composite = calculate_composite_score(newsapi_count, gnews_count, gtrends_score)
|
||||||
|
|
||||||
|
db.add(MemberTrendScore(
|
||||||
|
member_id=bioguide_id,
|
||||||
|
score_date=today,
|
||||||
|
newsapi_count=newsapi_count,
|
||||||
|
gnews_count=gnews_count,
|
||||||
|
gtrends_score=gtrends_score,
|
||||||
|
composite_score=composite,
|
||||||
|
))
|
||||||
|
db.commit()
|
||||||
|
logger.info(f"Scored member {bioguide_id}: composite={composite:.1f}")
|
||||||
|
return {"status": "ok", "composite": composite}
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
db.rollback()
|
||||||
|
logger.error(f"Member trend scoring failed for {bioguide_id}: {exc}")
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(bind=True, name="app.workers.member_interest.fetch_news_for_active_members")
|
||||||
|
def fetch_news_for_active_members(self):
|
||||||
|
"""
|
||||||
|
Scheduled task: fetch news for members who have been viewed or followed.
|
||||||
|
Prioritises members with detail_fetched set (profile has been viewed).
|
||||||
|
"""
|
||||||
|
db = get_sync_db()
|
||||||
|
try:
|
||||||
|
members = (
|
||||||
|
db.query(Member)
|
||||||
|
.filter(Member.detail_fetched.isnot(None))
|
||||||
|
.filter(Member.first_name.isnot(None))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for member in members:
|
||||||
|
fetch_member_news.delay(member.bioguide_id)
|
||||||
|
|
||||||
|
logger.info(f"Queued news fetch for {len(members)} members")
|
||||||
|
return {"queued": len(members)}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
@celery_app.task(bind=True, name="app.workers.member_interest.calculate_all_member_trend_scores")
|
||||||
|
def calculate_all_member_trend_scores(self):
|
||||||
|
"""
|
||||||
|
Scheduled nightly task: score all members that have been viewed.
|
||||||
|
Members are scored only after their profile has been loaded at least once.
|
||||||
|
"""
|
||||||
|
db = get_sync_db()
|
||||||
|
try:
|
||||||
|
members = (
|
||||||
|
db.query(Member)
|
||||||
|
.filter(Member.detail_fetched.isnot(None))
|
||||||
|
.filter(Member.first_name.isnot(None))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for member in members:
|
||||||
|
calculate_member_trend_score.delay(member.bioguide_id)
|
||||||
|
|
||||||
|
logger.info(f"Queued trend scoring for {len(members)} members")
|
||||||
|
return {"queued": len(members)}
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
@@ -41,9 +41,12 @@ def fetch_news_for_bill(self, bill_id: str):
|
|||||||
bill_number=bill.bill_number,
|
bill_number=bill.bill_number,
|
||||||
)
|
)
|
||||||
|
|
||||||
articles = news_service.fetch_newsapi_articles(query)
|
newsapi_articles = news_service.fetch_newsapi_articles(query)
|
||||||
|
gnews_articles = news_service.fetch_gnews_articles(query)
|
||||||
|
all_articles = newsapi_articles + gnews_articles
|
||||||
|
|
||||||
saved = 0
|
saved = 0
|
||||||
for article in articles:
|
for article in all_articles:
|
||||||
url = article.get("url")
|
url = article.get("url")
|
||||||
if not url:
|
if not url:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ import {
|
|||||||
FileText,
|
FileText,
|
||||||
Users,
|
Users,
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import { useMember, useMemberBills } from "@/lib/hooks/useMembers";
|
import { useMember, useMemberBills, useMemberTrend, useMemberNews } from "@/lib/hooks/useMembers";
|
||||||
|
import { TrendChart } from "@/components/bills/TrendChart";
|
||||||
|
import { NewsPanel } from "@/components/bills/NewsPanel";
|
||||||
import { FollowButton } from "@/components/shared/FollowButton";
|
import { FollowButton } from "@/components/shared/FollowButton";
|
||||||
import { BillCard } from "@/components/shared/BillCard";
|
import { BillCard } from "@/components/shared/BillCard";
|
||||||
import { cn, partyBadgeColor } from "@/lib/utils";
|
import { cn, partyBadgeColor } from "@/lib/utils";
|
||||||
@@ -28,6 +30,8 @@ export default function MemberDetailPage({ params }: { params: Promise<{ id: str
|
|||||||
const { id } = use(params);
|
const { id } = use(params);
|
||||||
const { data: member, isLoading } = useMember(id);
|
const { data: member, isLoading } = useMember(id);
|
||||||
const { data: billsData } = useMemberBills(id);
|
const { data: billsData } = useMemberBills(id);
|
||||||
|
const { data: trendData } = useMemberTrend(id, 30);
|
||||||
|
const { data: newsData } = useMemberNews(id);
|
||||||
|
|
||||||
if (isLoading) return <div className="text-center py-20 text-muted-foreground">Loading...</div>;
|
if (isLoading) return <div className="text-center py-20 text-muted-foreground">Loading...</div>;
|
||||||
if (!member) return <div className="text-center py-20 text-muted-foreground">Member not found.</div>;
|
if (!member) return <div className="text-center py-20 text-muted-foreground">Member not found.</div>;
|
||||||
@@ -156,6 +160,12 @@ export default function MemberDetailPage({ params }: { params: Promise<{ id: str
|
|||||||
|
|
||||||
{/* Right column */}
|
{/* Right column */}
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
|
{/* Public Interest */}
|
||||||
|
<TrendChart data={trendData ?? []} title="Public Interest" />
|
||||||
|
|
||||||
|
{/* News */}
|
||||||
|
<NewsPanel articles={newsData} />
|
||||||
|
|
||||||
{/* Legislation stats */}
|
{/* Legislation stats */}
|
||||||
{(member.sponsored_count != null || member.cosponsored_count != null) && (
|
{(member.sponsored_count != null || member.cosponsored_count != null) && (
|
||||||
<div className="bg-card border border-border rounded-lg p-4 space-y-3">
|
<div className="bg-card border border-border rounded-lg p-4 space-y-3">
|
||||||
|
|||||||
@@ -1,9 +1,16 @@
|
|||||||
import { ExternalLink, Newspaper } from "lucide-react";
|
import { ExternalLink, Newspaper } from "lucide-react";
|
||||||
import { NewsArticle } from "@/lib/types";
|
|
||||||
import { formatDate } from "@/lib/utils";
|
import { formatDate } from "@/lib/utils";
|
||||||
|
|
||||||
|
interface ArticleLike {
|
||||||
|
id: number;
|
||||||
|
source?: string;
|
||||||
|
headline?: string;
|
||||||
|
url?: string;
|
||||||
|
published_at?: string;
|
||||||
|
}
|
||||||
|
|
||||||
interface NewsPanelProps {
|
interface NewsPanelProps {
|
||||||
articles?: NewsArticle[];
|
articles?: ArticleLike[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export function NewsPanel({ articles }: NewsPanelProps) {
|
export function NewsPanel({ articles }: NewsPanelProps) {
|
||||||
|
|||||||
@@ -1,51 +1,89 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { TrendingUp } from "lucide-react";
|
import { TrendingUp, Newspaper, Radio } from "lucide-react";
|
||||||
import {
|
import {
|
||||||
LineChart,
|
ComposedChart,
|
||||||
Line,
|
Line,
|
||||||
|
Bar,
|
||||||
XAxis,
|
XAxis,
|
||||||
YAxis,
|
YAxis,
|
||||||
Tooltip,
|
Tooltip,
|
||||||
ResponsiveContainer,
|
ResponsiveContainer,
|
||||||
CartesianGrid,
|
CartesianGrid,
|
||||||
|
Legend,
|
||||||
} from "recharts";
|
} from "recharts";
|
||||||
import { TrendScore } from "@/lib/types";
|
import { TrendScore, MemberTrendScore } from "@/lib/types";
|
||||||
import { formatDate } from "@/lib/utils";
|
|
||||||
|
type AnyTrendScore = TrendScore | MemberTrendScore;
|
||||||
|
|
||||||
interface TrendChartProps {
|
interface TrendChartProps {
|
||||||
data?: TrendScore[];
|
data?: AnyTrendScore[];
|
||||||
|
title?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function TrendChart({ data }: TrendChartProps) {
|
function ScoreBadge({ label, value, icon }: { label: string; value: number | string; icon: React.ReactNode }) {
|
||||||
|
return (
|
||||||
|
<div className="flex flex-col items-center gap-0.5">
|
||||||
|
<div className="text-muted-foreground">{icon}</div>
|
||||||
|
<span className="text-xs font-semibold tabular-nums">{value}</span>
|
||||||
|
<span className="text-[10px] text-muted-foreground">{label}</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function TrendChart({ data, title = "Public Interest" }: TrendChartProps) {
|
||||||
const chartData = data?.map((d) => ({
|
const chartData = data?.map((d) => ({
|
||||||
date: new Date(d.score_date).toLocaleDateString("en-US", { month: "short", day: "numeric" }),
|
date: new Date(d.score_date + "T00:00:00").toLocaleDateString("en-US", { month: "short", day: "numeric" }),
|
||||||
score: Math.round(d.composite_score),
|
score: Math.round(d.composite_score),
|
||||||
news: d.newsapi_count,
|
newsapi: d.newsapi_count,
|
||||||
gnews: d.gnews_count,
|
gnews: d.gnews_count,
|
||||||
|
gtrends: Math.round(d.gtrends_score),
|
||||||
})) ?? [];
|
})) ?? [];
|
||||||
|
|
||||||
const latest = data?.[data.length - 1]?.composite_score;
|
const latest = data?.[data.length - 1];
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="bg-card border border-border rounded-lg p-4">
|
<div className="bg-card border border-border rounded-lg p-4 space-y-4">
|
||||||
<div className="flex items-center justify-between mb-4">
|
<div className="flex items-center justify-between">
|
||||||
<h3 className="font-semibold text-sm flex items-center gap-2">
|
<h3 className="font-semibold text-sm flex items-center gap-2">
|
||||||
<TrendingUp className="w-4 h-4" />
|
<TrendingUp className="w-4 h-4" />
|
||||||
Public Interest
|
{title}
|
||||||
</h3>
|
</h3>
|
||||||
{latest !== undefined && (
|
{latest !== undefined && (
|
||||||
<span className="text-2xl font-bold tabular-nums">{Math.round(latest)}</span>
|
<span className="text-2xl font-bold tabular-nums">{Math.round(latest.composite_score)}</span>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Signal breakdown badges */}
|
||||||
|
{latest && (
|
||||||
|
<div className="flex justify-around border border-border rounded-md p-2 bg-muted/30">
|
||||||
|
<ScoreBadge
|
||||||
|
label="NewsAPI"
|
||||||
|
value={latest.newsapi_count}
|
||||||
|
icon={<Newspaper className="w-3 h-3" />}
|
||||||
|
/>
|
||||||
|
<div className="w-px bg-border" />
|
||||||
|
<ScoreBadge
|
||||||
|
label="Google News"
|
||||||
|
value={latest.gnews_count}
|
||||||
|
icon={<Radio className="w-3 h-3" />}
|
||||||
|
/>
|
||||||
|
<div className="w-px bg-border" />
|
||||||
|
<ScoreBadge
|
||||||
|
label="Trends"
|
||||||
|
value={`${Math.round(latest.gtrends_score)}/100`}
|
||||||
|
icon={<TrendingUp className="w-3 h-3" />}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
{chartData.length === 0 ? (
|
{chartData.length === 0 ? (
|
||||||
<p className="text-xs text-muted-foreground italic text-center py-8">
|
<p className="text-xs text-muted-foreground italic text-center py-8">
|
||||||
Trend data not yet available.
|
Interest data not yet available. Check back after the nightly scoring run.
|
||||||
</p>
|
</p>
|
||||||
) : (
|
) : (
|
||||||
<ResponsiveContainer width="100%" height={180}>
|
<ResponsiveContainer width="100%" height={180}>
|
||||||
<LineChart data={chartData} margin={{ top: 4, right: 4, left: -20, bottom: 0 }}>
|
<ComposedChart data={chartData} margin={{ top: 4, right: 4, left: -20, bottom: 0 }}>
|
||||||
<CartesianGrid strokeDasharray="3 3" stroke="hsl(var(--border))" />
|
<CartesianGrid strokeDasharray="3 3" stroke="hsl(var(--border))" />
|
||||||
<XAxis
|
<XAxis
|
||||||
dataKey="date"
|
dataKey="date"
|
||||||
@@ -64,23 +102,33 @@ export function TrendChart({ data }: TrendChartProps) {
|
|||||||
borderRadius: "6px",
|
borderRadius: "6px",
|
||||||
fontSize: "12px",
|
fontSize: "12px",
|
||||||
}}
|
}}
|
||||||
|
formatter={(value: number, name: string) => {
|
||||||
|
const labels: Record<string, string> = {
|
||||||
|
score: "Composite",
|
||||||
|
newsapi: "NewsAPI articles",
|
||||||
|
gnews: "Google News articles",
|
||||||
|
gtrends: "Google Trends",
|
||||||
|
};
|
||||||
|
return [value, labels[name] ?? name];
|
||||||
|
}}
|
||||||
/>
|
/>
|
||||||
|
<Bar dataKey="gnews" fill="hsl(var(--muted-foreground))" opacity={0.3} name="gnews" radius={[2, 2, 0, 0]} />
|
||||||
|
<Bar dataKey="newsapi" fill="hsl(var(--primary))" opacity={0.3} name="newsapi" radius={[2, 2, 0, 0]} />
|
||||||
<Line
|
<Line
|
||||||
type="monotone"
|
type="monotone"
|
||||||
dataKey="score"
|
dataKey="score"
|
||||||
stroke="hsl(var(--primary))"
|
stroke="hsl(var(--primary))"
|
||||||
strokeWidth={2}
|
strokeWidth={2}
|
||||||
dot={false}
|
dot={false}
|
||||||
name="Zeitgeist Score"
|
name="score"
|
||||||
/>
|
/>
|
||||||
</LineChart>
|
</ComposedChart>
|
||||||
</ResponsiveContainer>
|
</ResponsiveContainer>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
<div className="mt-3 flex gap-4 text-xs text-muted-foreground">
|
<p className="text-[10px] text-muted-foreground">
|
||||||
<span>Score: 0–100 composite</span>
|
Composite 0–100 · NewsAPI articles (max 40 pts) + Google News volume (max 30 pts) + Google Trends score (max 30 pts)
|
||||||
<span>NewsAPI + Google News + Trends</span>
|
</p>
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import type {
|
|||||||
DashboardData,
|
DashboardData,
|
||||||
Follow,
|
Follow,
|
||||||
Member,
|
Member,
|
||||||
|
MemberTrendScore,
|
||||||
|
MemberNewsArticle,
|
||||||
NewsArticle,
|
NewsArticle,
|
||||||
PaginatedResponse,
|
PaginatedResponse,
|
||||||
SettingsData,
|
SettingsData,
|
||||||
@@ -81,6 +83,10 @@ export const membersAPI = {
|
|||||||
apiClient.get<Member>(`/api/members/${id}`).then((r) => r.data),
|
apiClient.get<Member>(`/api/members/${id}`).then((r) => r.data),
|
||||||
getBills: (id: string, params?: Record<string, unknown>) =>
|
getBills: (id: string, params?: Record<string, unknown>) =>
|
||||||
apiClient.get<PaginatedResponse<Bill>>(`/api/members/${id}/bills`, { params }).then((r) => r.data),
|
apiClient.get<PaginatedResponse<Bill>>(`/api/members/${id}/bills`, { params }).then((r) => r.data),
|
||||||
|
getTrend: (id: string, days?: number) =>
|
||||||
|
apiClient.get<MemberTrendScore[]>(`/api/members/${id}/trend`, { params: { days } }).then((r) => r.data),
|
||||||
|
getNews: (id: string) =>
|
||||||
|
apiClient.get<MemberNewsArticle[]>(`/api/members/${id}/news`).then((r) => r.data),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Follows
|
// Follows
|
||||||
|
|||||||
@@ -26,3 +26,21 @@ export function useMemberBills(id: string) {
|
|||||||
enabled: !!id,
|
enabled: !!id,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function useMemberTrend(id: string, days = 30) {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: ["member-trend", id, days],
|
||||||
|
queryFn: () => membersAPI.getTrend(id, days),
|
||||||
|
staleTime: 60 * 60 * 1000,
|
||||||
|
enabled: !!id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function useMemberNews(id: string) {
|
||||||
|
return useQuery({
|
||||||
|
queryKey: ["member-news", id],
|
||||||
|
queryFn: () => membersAPI.getNews(id),
|
||||||
|
staleTime: 10 * 60 * 1000,
|
||||||
|
enabled: !!id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|||||||
@@ -15,6 +15,23 @@ export interface MemberLeadership {
|
|||||||
current?: boolean;
|
current?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface MemberTrendScore {
|
||||||
|
score_date: string;
|
||||||
|
newsapi_count: number;
|
||||||
|
gnews_count: number;
|
||||||
|
gtrends_score: number;
|
||||||
|
composite_score: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MemberNewsArticle {
|
||||||
|
id: number;
|
||||||
|
source?: string;
|
||||||
|
headline?: string;
|
||||||
|
url?: string;
|
||||||
|
published_at?: string;
|
||||||
|
relevance_score?: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface Member {
|
export interface Member {
|
||||||
bioguide_id: string;
|
bioguide_id: string;
|
||||||
name: string;
|
name: string;
|
||||||
@@ -34,6 +51,7 @@ export interface Member {
|
|||||||
leadership_json?: MemberLeadership[];
|
leadership_json?: MemberLeadership[];
|
||||||
sponsored_count?: number;
|
sponsored_count?: number;
|
||||||
cosponsored_count?: number;
|
cosponsored_count?: number;
|
||||||
|
latest_trend?: MemberTrendScore;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface CitedPoint {
|
export interface CitedPoint {
|
||||||
|
|||||||
Reference in New Issue
Block a user