"""Celery tasks for organization source extensions.""" from __future__ import annotations import logging import math from dataclasses import asdict from apps.core.tasks import PeriodicTask as CorePeriodicTask from celery import shared_task from celery.exceptions import MaxRetriesExceededError from django.conf import settings from django.core.cache import cache from organizations.cache import invalidate_organization_api_cache from organizations.source_backfill import OrganizationSourceBackfillService from organizations.source_record_export import build_source_record_export_artifacts logger = logging.getLogger(__name__) @shared_task(bind=True, base=CorePeriodicTask) def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 """Build and atomically publish the nightly source-record export matrix.""" return _refresh_source_record_export_artifacts(self, all_years=False) @shared_task(bind=True, base=CorePeriodicTask) def refresh_all_years_source_record_export_artifacts(self) -> dict: """Publish all available years independently of the annual generation.""" return _refresh_source_record_export_artifacts(self, all_years=True) def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict: lock_key = getattr( settings, "SOURCE_RECORD_EXPORT_LOCK_KEY", "organizations:source-record-exports:lock", ) lock_ttl = int( getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60) ) if not cache.add(lock_key, "1", timeout=lock_ttl): # Every successful registry publication requests a rebuild. Dropping a # request here can leave the in-flight generation stale until tomorrow. # Retry beyond one full lease so a killed builder's lock can expire too. countdown = 30 max_retries = max(1, math.ceil(lock_ttl / countdown) + 2) try: raise task.retry(countdown=countdown, max_retries=max_retries) except MaxRetriesExceededError: logger.error( "Source-record export retry budget exhausted while waiting for build lock" ) raise try: generation = build_source_record_export_artifacts(all_years=all_years) result = { "status": "success", "generation_id": generation.generation_id, "generated_at": generation.generated_at, "export_year": generation.export_year, "artifacts_count": generation.artifacts_count, "files_count": generation.files_count, "records_count": generation.records_count, "total_size": generation.total_size, } logger.info("Source-record export generation published: %s", result) return result finally: cache.delete(lock_key) @shared_task def backfill_all_organization_sources(batch_size: int = 100) -> dict: """Backfill all organization source extensions from legacy parser tables.""" result = OrganizationSourceBackfillService.backfill() invalidate_organization_api_cache() payload = {"batch_size": batch_size, **asdict(result)} logger.info("All organization source extensions backfilled: %s", payload) return payload @shared_task def backfill_organization_sources_for_parser_batch( *, source: str, batch_id: int, batch_size: int = 100, ) -> dict: """Backfill source extensions affected by one parser batch.""" result = OrganizationSourceBackfillService.backfill( source=source, batch_id=batch_id, ) invalidate_organization_api_cache() payload = { "source": source, "batch_id": batch_id, "batch_size": batch_size, **asdict(result), } logger.info("Organization source extensions backfilled: %s", payload) return payload @shared_task def refresh_all_organization_data_snapshots(batch_size: int = 100) -> dict: """Deprecated compatibility wrapper for the old snapshot schedule.""" return backfill_all_organization_sources(batch_size=batch_size) @shared_task def refresh_organization_data_snapshots_for_parser_batch( *, source: str, batch_id: int, batch_size: int = 100, ) -> dict: """Deprecated compatibility wrapper for old parser task imports.""" return backfill_organization_sources_for_parser_batch( source=source, batch_id=batch_id, batch_size=batch_size, )