127 lines
4.4 KiB
Python
127 lines
4.4 KiB
Python
"""Celery tasks for organization source extensions."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import math
|
|
from dataclasses import asdict
|
|
|
|
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
|
from celery import shared_task
|
|
from celery.exceptions import MaxRetriesExceededError
|
|
from django.conf import settings
|
|
from django.core.cache import cache
|
|
|
|
from organizations.cache import invalidate_organization_api_cache
|
|
from organizations.source_backfill import OrganizationSourceBackfillService
|
|
from organizations.source_record_export import build_source_record_export_artifacts
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@shared_task(bind=True, base=CorePeriodicTask)
|
|
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
|
"""Build and atomically publish the nightly source-record export matrix."""
|
|
return _refresh_source_record_export_artifacts(self, all_years=False)
|
|
|
|
|
|
@shared_task(bind=True, base=CorePeriodicTask)
|
|
def refresh_all_years_source_record_export_artifacts(self) -> dict:
|
|
"""Publish all available years independently of the annual generation."""
|
|
return _refresh_source_record_export_artifacts(self, all_years=True)
|
|
|
|
|
|
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
|
|
lock_key = getattr(
|
|
settings,
|
|
"SOURCE_RECORD_EXPORT_LOCK_KEY",
|
|
"organizations:source-record-exports:lock",
|
|
)
|
|
lock_ttl = int(
|
|
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
|
|
)
|
|
if not cache.add(lock_key, "1", timeout=lock_ttl):
|
|
# Every successful registry publication requests a rebuild. Dropping a
|
|
# request here can leave the in-flight generation stale until tomorrow.
|
|
# Retry beyond one full lease so a killed builder's lock can expire too.
|
|
countdown = 30
|
|
max_retries = max(1, math.ceil(lock_ttl / countdown) + 2)
|
|
try:
|
|
raise task.retry(countdown=countdown, max_retries=max_retries)
|
|
except MaxRetriesExceededError:
|
|
logger.error(
|
|
"Source-record export retry budget exhausted while waiting for build lock"
|
|
)
|
|
raise
|
|
|
|
try:
|
|
generation = build_source_record_export_artifacts(all_years=all_years)
|
|
result = {
|
|
"status": "success",
|
|
"generation_id": generation.generation_id,
|
|
"generated_at": generation.generated_at,
|
|
"export_year": generation.export_year,
|
|
"artifacts_count": generation.artifacts_count,
|
|
"files_count": generation.files_count,
|
|
"records_count": generation.records_count,
|
|
"total_size": generation.total_size,
|
|
}
|
|
logger.info("Source-record export generation published: %s", result)
|
|
return result
|
|
finally:
|
|
cache.delete(lock_key)
|
|
|
|
|
|
@shared_task
|
|
def backfill_all_organization_sources(batch_size: int = 100) -> dict:
|
|
"""Backfill all organization source extensions from legacy parser tables."""
|
|
result = OrganizationSourceBackfillService.backfill()
|
|
invalidate_organization_api_cache()
|
|
payload = {"batch_size": batch_size, **asdict(result)}
|
|
logger.info("All organization source extensions backfilled: %s", payload)
|
|
return payload
|
|
|
|
|
|
@shared_task
|
|
def backfill_organization_sources_for_parser_batch(
|
|
*,
|
|
source: str,
|
|
batch_id: int,
|
|
batch_size: int = 100,
|
|
) -> dict:
|
|
"""Backfill source extensions affected by one parser batch."""
|
|
result = OrganizationSourceBackfillService.backfill(
|
|
source=source,
|
|
batch_id=batch_id,
|
|
)
|
|
invalidate_organization_api_cache()
|
|
payload = {
|
|
"source": source,
|
|
"batch_id": batch_id,
|
|
"batch_size": batch_size,
|
|
**asdict(result),
|
|
}
|
|
logger.info("Organization source extensions backfilled: %s", payload)
|
|
return payload
|
|
|
|
|
|
@shared_task
|
|
def refresh_all_organization_data_snapshots(batch_size: int = 100) -> dict:
|
|
"""Deprecated compatibility wrapper for the old snapshot schedule."""
|
|
return backfill_all_organization_sources(batch_size=batch_size)
|
|
|
|
|
|
@shared_task
|
|
def refresh_organization_data_snapshots_for_parser_batch(
|
|
*,
|
|
source: str,
|
|
batch_id: int,
|
|
batch_size: int = 100,
|
|
) -> dict:
|
|
"""Deprecated compatibility wrapper for old parser task imports."""
|
|
return backfill_organization_sources_for_parser_batch(
|
|
source=source,
|
|
batch_id=batch_id,
|
|
batch_size=batch_size,
|
|
)
|