feat: API optimizations — quota batching, ETags, caching, async sponsor (v0.9.7)

Nine efficiency improvements across the data pipeline:

1. NewsAPI OR batching (news_service.py + news_fetcher.py)
   - Combine up to 4 bills per NewsAPI call using OR query syntax
   - NEWSAPI_BATCH_SIZE=4 means ~4× effective daily quota (100→400 bill-fetches)
   - fetch_news_for_bill_batch task; fetch_news_for_active_bills queues batches

2. Google News RSS cache (news_service.py)
   - 2-hour Redis cache shared between news_fetcher and trend_scorer
   - Eliminates duplicate RSS hits when both workers run against same bill
   - clear_gnews_cache() admin helper + admin endpoint

3. pytrends keyword batching (trends_service.py + trend_scorer.py)
   - Compare up to 5 bills per pytrends call instead of 1
   - get_trends_scores_batch() returns scores in original order
   - Reduces pytrends calls by ~5× and associated rate-limit risk

4. GovInfo ETags (govinfo_api.py + document_fetcher.py)
   - If-None-Match conditional GET; DocumentUnchangedError on HTTP 304
   - ETags stored in Redis (30-day TTL) keyed by MD5(url)
   - document_fetcher catches DocumentUnchangedError → {"status": "unchanged"}

5. Anthropic prompt caching (llm_service.py)
   - cache_control: {type: ephemeral} on system messages in AnthropicProvider
   - Caches the ~700-token system prompt server-side; ~50% cost reduction on
     repeated calls within the 5-minute cache window

6. Async sponsor fetch (congress_poller.py)
   - New fetch_sponsor_for_bill Celery task replaces blocking get_bill_detail()
     inline in poll loop
   - Bills saved immediately with sponsor_id=None; sponsor linked async
   - Removes 0.25s sleep per new bill from poll hot path

7. Skip doc fetch for procedural actions (congress_poller.py)
   - _DOC_PRODUCING_CATEGORIES = {vote, committee_report, presidential, ...}
   - fetch_bill_documents only enqueued when action is likely to produce
     new GovInfo text (saves ~60–70% of unnecessary document fetch attempts)

8. Adaptive poll frequency (congress_poller.py)
   - _is_congress_off_hours(): weekends + before 9AM / after 9PM EST
   - Skips poll if off-hours AND last poll < 1 hour ago
   - Prevents wasteful polling when Congress is not in session

9. Admin panel additions (admin.py + settings/page.tsx + api.ts)
   - GET /api/admin/newsapi-quota → remaining calls today
   - POST /api/admin/clear-gnews-cache → flush RSS cache
   - Settings page shows NewsAPI quota remaining (amber if < 10)
   - "Clear Google News Cache" button in Manual Controls

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/app/services/news_service.py

@@ -4,6 +4,8 @@ News correlation service.
 - NewsAPI.org: structured news articles per bill (100 req/day limit)
 - Google News RSS: volume signal for zeitgeist scoring (no limit)
 """
+import hashlib
+import json
 import logging
 import time
 import urllib.parse
@@ -22,11 +24,13 @@ logger = logging.getLogger(__name__)
 NEWSAPI_BASE = "https://newsapi.org/v2"
 GOOGLE_NEWS_RSS = "https://news.google.com/rss/search"
 NEWSAPI_DAILY_LIMIT = 95  # Leave 5 as buffer
+NEWSAPI_BATCH_SIZE = 4    # Bills per OR-combined API call
 
 _NEWSAPI_REDIS_PREFIX = "newsapi:daily_calls:"
+_GNEWS_CACHE_TTL = 7200  # 2 hours — both trend_scorer and news_fetcher share cache
 
 
-def _newsapi_redis():
+def _redis():
     return redis.from_url(settings.REDIS_URL, decode_responses=True)
 
 
@@ -34,7 +38,7 @@ def _newsapi_quota_ok() -> bool:
     """Return True if we have quota remaining for today."""
     try:
         key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
-        used = int(_newsapi_redis().get(key) or 0)
+        used = int(_redis().get(key) or 0)
         return used < NEWSAPI_DAILY_LIMIT
     except Exception:
         return True  # Don't block on Redis errors
@@ -42,7 +46,7 @@ def _newsapi_quota_ok() -> bool:
 
 def _newsapi_record_call():
     try:
-        r = _newsapi_redis()
+        r = _redis()
         key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
         pipe = r.pipeline()
         pipe.incr(key)
@@ -52,6 +56,28 @@ def _newsapi_record_call():
         pass
 
 
+def get_newsapi_quota_remaining() -> int:
+    """Return the number of NewsAPI calls still available today."""
+    try:
+        key = f"{_NEWSAPI_REDIS_PREFIX}{date.today().isoformat()}"
+        used = int(_redis().get(key) or 0)
+        return max(0, NEWSAPI_DAILY_LIMIT - used)
+    except Exception:
+        return NEWSAPI_DAILY_LIMIT
+
+
+def clear_gnews_cache() -> int:
+    """Delete all cached Google News RSS results. Returns number of keys deleted."""
+    try:
+        r = _redis()
+        keys = r.keys("gnews:*")
+        if keys:
+            return r.delete(*keys)
+        return 0
+    except Exception:
+        return 0
+
+
 @retry(stop=stop_after_attempt(2), wait=wait_exponential(min=1, max=5))
 def _newsapi_get(endpoint: str, params: dict) -> dict:
     params["apiKey"] = settings.NEWSAPI_KEY
@@ -109,8 +135,85 @@ def fetch_newsapi_articles(query: str, days: int = 30) -> list[dict]:
         return []
 
 
+def fetch_newsapi_articles_batch(
+    bill_queries: list[tuple[str, str]],
+    days: int = 30,
+) -> dict[str, list[dict]]:
+    """
+    Fetch NewsAPI articles for up to NEWSAPI_BATCH_SIZE bills in ONE API call
+    using OR syntax.  Returns {bill_id: [articles]} — each article attributed
+    to the bill whose query terms appear in the headline/description.
+    """
+    empty = {bill_id: [] for bill_id, _ in bill_queries}
+    if not settings.NEWSAPI_KEY or not bill_queries:
+        return empty
+    if not _newsapi_quota_ok():
+        logger.warning("NewsAPI daily quota exhausted — skipping batch fetch")
+        return empty
+
+    combined_q = " OR ".join(q for _, q in bill_queries)
+    try:
+        from_date = (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
+        data = _newsapi_get("everything", {
+            "q": combined_q,
+            "language": "en",
+            "sortBy": "relevancy",
+            "pageSize": 20,
+            "from": from_date,
+        })
+        _newsapi_record_call()
+        articles = data.get("articles", [])
+
+        result: dict[str, list[dict]] = {bill_id: [] for bill_id, _ in bill_queries}
+        for article in articles:
+            content = " ".join([
+                article.get("title", ""),
+                article.get("description", "") or "",
+            ]).lower()
+            for bill_id, query in bill_queries:
+                # Match if any meaningful term from this bill's query appears in the article
+                terms = [t.strip('" ').lower() for t in query.split(" OR ")]
+                if any(len(t) > 3 and t in content for t in terms):
+                    result[bill_id].append({
+                        "source": article.get("source", {}).get("name", ""),
+                        "headline": article.get("title", ""),
+                        "url": article.get("url", ""),
+                        "published_at": article.get("publishedAt"),
+                    })
+        return result
+    except Exception as e:
+        logger.error(f"NewsAPI batch fetch failed: {e}")
+        return empty
+
+
+# ── Google News RSS ─────────────────────────────────────────────────────────────
+
+def _gnews_cache_key(query: str, kind: str, days: int) -> str:
+    h = hashlib.md5(f"{query}:{days}".encode()).hexdigest()[:12]
+    return f"gnews:{kind}:{h}"
+
+
 def fetch_gnews_count(query: str, days: int = 30) -> int:
-    """Count articles in Google News RSS for the past N days. Used as volume signal."""
+    """Count articles in Google News RSS. Results cached in Redis for 2 hours."""
+    cache_key = _gnews_cache_key(query, "count", days)
+    try:
+        cached = _redis().get(cache_key)
+        if cached is not None:
+            return int(cached)
+    except Exception:
+        pass
+
+    count = _fetch_gnews_count_raw(query, days)
+
+    try:
+        _redis().setex(cache_key, _GNEWS_CACHE_TTL, count)
+    except Exception:
+        pass
+    return count
+
+
+def _fetch_gnews_count_raw(query: str, days: int) -> int:
+    """Fetch gnews article count directly (no cache)."""
     try:
         encoded = urllib.parse.quote(f"{query} when:{days}d")
         url = f"{GOOGLE_NEWS_RSS}?q={encoded}&hl=en-US&gl=US&ceid=US:en"
@@ -124,11 +227,9 @@ def fetch_gnews_count(query: str, days: int = 30) -> int:
 
 def _gnews_entry_url(entry) -> str:
     """Extract the article URL from a feedparser Google News RSS entry."""
-    # Primary: entry.link attribute
     link = getattr(entry, "link", None) or entry.get("link", "")
     if link:
         return link
-    # Fallback: scan entry.links list for rel=alternate
     for lnk in getattr(entry, "links", []):
         href = lnk.get("href", "")
         if href:
@@ -137,7 +238,27 @@ def _gnews_entry_url(entry) -> str:
 
 
 def fetch_gnews_articles(query: str, days: int = 30) -> list[dict]:
-    """Fetch articles from Google News RSS. No rate limit — unlimited source."""
+    """Fetch articles from Google News RSS. Results cached in Redis for 2 hours."""
+    import time as time_mod
+    cache_key = _gnews_cache_key(query, "articles", days)
+    try:
+        cached = _redis().get(cache_key)
+        if cached is not None:
+            return json.loads(cached)
+    except Exception:
+        pass
+
+    articles = _fetch_gnews_articles_raw(query, days)
+
+    try:
+        _redis().setex(cache_key, _GNEWS_CACHE_TTL, json.dumps(articles))
+    except Exception:
+        pass
+    return articles
+
+
+def _fetch_gnews_articles_raw(query: str, days: int) -> list[dict]:
+    """Fetch gnews articles directly (no cache)."""
     import time as time_mod
     try:
         encoded = urllib.parse.quote(f"{query} when:{days}d")
@@ -154,7 +275,6 @@ def fetch_gnews_articles(query: str, days: int = 30) -> list[dict]:
                     ).isoformat()
                 except Exception:
                     pass
-            # Source: feedparser puts it in entry.source.title for Google News
             source = ""
             src = getattr(entry, "source", None)
             if src: