f6770b16be
vote_fetcher was missing from Celery's include list (task not registered with workers) and had no beat schedule — votes only fetched on-demand when a user visited a bill's votes page. Stanced bills (pocket_veto/pocket_boost) never had votes fetched, leaving the alignment page blank. Add fetch_votes_for_stanced_bills nightly task (4:30 AM UTC) that queues fetch_bill_votes for every bill any user has stanced but has no stored votes. Register vote_fetcher in the include list and add it to the polling queue route. Authored by: Jack Levy
113 lines
4.6 KiB
Python
113 lines
4.6 KiB
Python
from celery import Celery
|
|
from celery.schedules import crontab
|
|
from kombu import Queue
|
|
|
|
from app.config import settings
|
|
|
|
celery_app = Celery(
|
|
"pocketveto",
|
|
broker=settings.REDIS_URL,
|
|
backend=settings.REDIS_URL,
|
|
include=[
|
|
"app.workers.congress_poller",
|
|
"app.workers.document_fetcher",
|
|
"app.workers.llm_processor",
|
|
"app.workers.news_fetcher",
|
|
"app.workers.trend_scorer",
|
|
"app.workers.member_interest",
|
|
"app.workers.notification_dispatcher",
|
|
"app.workers.llm_batch_processor",
|
|
"app.workers.bill_classifier",
|
|
"app.workers.vote_fetcher",
|
|
],
|
|
)
|
|
|
|
celery_app.conf.update(
|
|
task_serializer="json",
|
|
result_serializer="json",
|
|
accept_content=["json"],
|
|
timezone="UTC",
|
|
enable_utc=True,
|
|
# Late ack: task is only removed from queue after completion, not on pickup.
|
|
# Combined with idempotent tasks, this ensures no work is lost if a worker crashes.
|
|
task_acks_late=True,
|
|
# Prevent workers from prefetching LLM tasks and blocking other workers.
|
|
worker_prefetch_multiplier=1,
|
|
# Route tasks to named queues
|
|
task_routes={
|
|
"app.workers.congress_poller.*": {"queue": "polling"},
|
|
"app.workers.document_fetcher.*": {"queue": "documents"},
|
|
"app.workers.llm_processor.*": {"queue": "llm"},
|
|
"app.workers.llm_batch_processor.*": {"queue": "llm"},
|
|
"app.workers.bill_classifier.*": {"queue": "llm"},
|
|
"app.workers.news_fetcher.*": {"queue": "news"},
|
|
"app.workers.trend_scorer.*": {"queue": "news"},
|
|
"app.workers.member_interest.*": {"queue": "news"},
|
|
"app.workers.notification_dispatcher.*": {"queue": "polling"},
|
|
"app.workers.vote_fetcher.*": {"queue": "polling"},
|
|
},
|
|
task_queues=[
|
|
Queue("polling"),
|
|
Queue("documents"),
|
|
Queue("llm"),
|
|
Queue("news"),
|
|
],
|
|
# RedBeat stores schedule in Redis — restart-safe and dynamically updatable
|
|
redbeat_redis_url=settings.REDIS_URL,
|
|
beat_scheduler="redbeat.RedBeatScheduler",
|
|
beat_schedule={
|
|
"poll-congress-bills": {
|
|
"task": "app.workers.congress_poller.poll_congress_bills",
|
|
"schedule": crontab(minute=f"*/{settings.CONGRESS_POLL_INTERVAL_MINUTES}"),
|
|
},
|
|
"fetch-news-active-bills": {
|
|
"task": "app.workers.news_fetcher.fetch_news_for_active_bills",
|
|
"schedule": crontab(hour="*/6", minute=0),
|
|
},
|
|
"calculate-trend-scores": {
|
|
"task": "app.workers.trend_scorer.calculate_all_trend_scores",
|
|
"schedule": crontab(hour=2, minute=0),
|
|
},
|
|
"fetch-news-active-members": {
|
|
"task": "app.workers.member_interest.fetch_news_for_active_members",
|
|
"schedule": crontab(hour="*/12", minute=30),
|
|
},
|
|
"calculate-member-trend-scores": {
|
|
"task": "app.workers.member_interest.calculate_all_member_trend_scores",
|
|
"schedule": crontab(hour=3, minute=0),
|
|
},
|
|
"sync-members": {
|
|
"task": "app.workers.congress_poller.sync_members",
|
|
"schedule": crontab(hour=1, minute=0), # 1 AM UTC daily — refreshes chamber/district/contact info
|
|
},
|
|
"fetch-actions-active-bills": {
|
|
"task": "app.workers.congress_poller.fetch_actions_for_active_bills",
|
|
"schedule": crontab(hour=4, minute=0), # 4 AM UTC, after trend + member scoring
|
|
},
|
|
"fetch-votes-for-stanced-bills": {
|
|
"task": "app.workers.vote_fetcher.fetch_votes_for_stanced_bills",
|
|
"schedule": crontab(hour=4, minute=30), # 4:30 AM UTC daily
|
|
},
|
|
"dispatch-notifications": {
|
|
"task": "app.workers.notification_dispatcher.dispatch_notifications",
|
|
"schedule": crontab(minute="*/5"), # Every 5 minutes
|
|
},
|
|
"send-notification-digest": {
|
|
"task": "app.workers.notification_dispatcher.send_notification_digest",
|
|
"schedule": crontab(hour=8, minute=0), # 8 AM UTC daily
|
|
},
|
|
"send-weekly-digest": {
|
|
"task": "app.workers.notification_dispatcher.send_weekly_digest",
|
|
"schedule": crontab(hour=8, minute=30, day_of_week=1), # Monday 8:30 AM UTC
|
|
},
|
|
"poll-llm-batch-results": {
|
|
"task": "app.workers.llm_batch_processor.poll_llm_batch_results",
|
|
"schedule": crontab(minute="*/30"),
|
|
},
|
|
"calculate-effectiveness-scores": {
|
|
"task": "app.workers.bill_classifier.calculate_effectiveness_scores",
|
|
"schedule": crontab(hour=5, minute=0), # 5 AM UTC, after all other nightly tasks
|
|
},
|
|
},
|
|
)
|