a66b5b4bcb
Adds Google Trends, NewsAPI, and Google News RSS scoring for members,
mirroring the existing bill interest pipeline. Member profiles now show
a Public Interest chart (with signal breakdown) and a Related News panel.
Key changes:
- New member_trend_scores + member_news_articles tables (migration 0008)
- fetch_gnews_articles() added to news_service for unlimited RSS article storage
- Bill news fetcher now combines NewsAPI + Google News RSS (more coverage)
- New member_interest Celery worker with scheduled news + trend tasks
- GET /members/{id}/trend and /news API endpoints
- TrendChart redesigned with signal breakdown badges and bar+line combo chart
- NewsPanel accepts generic article shape (bills and members)
Co-Authored-By: Jack Levy
73 lines
2.1 KiB
Python
73 lines
2.1 KiB
Python
"""
|
||
Google Trends service (via pytrends).
|
||
|
||
pytrends is unofficial web scraping — Google blocks it sporadically.
|
||
All calls are wrapped in try/except and return 0 on any failure.
|
||
"""
|
||
import logging
|
||
import random
|
||
import time
|
||
|
||
from app.config import settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def get_trends_score(keywords: list[str]) -> float:
|
||
"""
|
||
Return a 0–100 interest score for the given keywords over the past 90 days.
|
||
Returns 0.0 on any failure (rate limit, empty data, exception).
|
||
"""
|
||
if not settings.PYTRENDS_ENABLED or not keywords:
|
||
return 0.0
|
||
try:
|
||
from pytrends.request import TrendReq
|
||
|
||
# Jitter to avoid detection as bot
|
||
time.sleep(random.uniform(2.0, 5.0))
|
||
|
||
pytrends = TrendReq(hl="en-US", tz=0, timeout=(10, 25))
|
||
kw_list = [k for k in keywords[:5] if k] # max 5 keywords
|
||
if not kw_list:
|
||
return 0.0
|
||
|
||
pytrends.build_payload(kw_list, timeframe="today 3-m", geo="US")
|
||
data = pytrends.interest_over_time()
|
||
|
||
if data is None or data.empty:
|
||
return 0.0
|
||
|
||
# Average the most recent 14 data points for the primary keyword
|
||
primary = kw_list[0]
|
||
if primary not in data.columns:
|
||
return 0.0
|
||
|
||
recent = data[primary].tail(14)
|
||
return float(recent.mean())
|
||
|
||
except Exception as e:
|
||
logger.debug(f"pytrends failed (non-critical): {e}")
|
||
return 0.0
|
||
|
||
|
||
def keywords_for_member(first_name: str, last_name: str) -> list[str]:
|
||
"""Extract meaningful search keywords for a member of Congress."""
|
||
full_name = f"{first_name} {last_name}".strip()
|
||
if not full_name:
|
||
return []
|
||
return [full_name]
|
||
|
||
|
||
def keywords_for_bill(title: str, short_title: str, topic_tags: list[str]) -> list[str]:
|
||
"""Extract meaningful search keywords for a bill."""
|
||
keywords = []
|
||
if short_title:
|
||
keywords.append(short_title)
|
||
elif title:
|
||
# Use first 5 words of title
|
||
words = title.split()[:5]
|
||
if len(words) >= 2:
|
||
keywords.append(" ".join(words))
|
||
keywords.extend(tag.replace("-", " ") for tag in (topic_tags or [])[:3])
|
||
return keywords[:5]
|