Files
mostovik-backend/src/organizations/tasks.py
2026-09-28 12:06:50 +02:00

127 lines
4.4 KiB
Python

"""Celery tasks for organization source extensions."""
from __future__ import annotations
import logging
import math
from dataclasses import asdict
from apps.core.tasks import PeriodicTask as CorePeriodicTask
from celery import shared_task
from celery.exceptions import MaxRetriesExceededError
from django.conf import settings
from django.core.cache import cache
from organizations.cache import invalidate_organization_api_cache
from organizations.source_backfill import OrganizationSourceBackfillService
from organizations.source_record_export import build_source_record_export_artifacts
logger = logging.getLogger(__name__)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
"""Build and atomically publish the nightly source-record export matrix."""
return _refresh_source_record_export_artifacts(self, all_years=False)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_all_years_source_record_export_artifacts(self) -> dict:
"""Publish all available years independently of the annual generation."""
return _refresh_source_record_export_artifacts(self, all_years=True)
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
lock_key = getattr(
settings,
"SOURCE_RECORD_EXPORT_LOCK_KEY",
"organizations:source-record-exports:lock",
)
lock_ttl = int(
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
)
if not cache.add(lock_key, "1", timeout=lock_ttl):
# Every successful registry publication requests a rebuild. Dropping a
# request here can leave the in-flight generation stale until tomorrow.
# Retry beyond one full lease so a killed builder's lock can expire too.
countdown = 30
max_retries = max(1, math.ceil(lock_ttl / countdown) + 2)
try:
raise task.retry(countdown=countdown, max_retries=max_retries)
except MaxRetriesExceededError:
logger.error(
"Source-record export retry budget exhausted while waiting for build lock"
)
raise
try:
generation = build_source_record_export_artifacts(all_years=all_years)
result = {
"status": "success",
"generation_id": generation.generation_id,
"generated_at": generation.generated_at,
"export_year": generation.export_year,
"artifacts_count": generation.artifacts_count,
"files_count": generation.files_count,
"records_count": generation.records_count,
"total_size": generation.total_size,
}
logger.info("Source-record export generation published: %s", result)
return result
finally:
cache.delete(lock_key)
@shared_task
def backfill_all_organization_sources(batch_size: int = 100) -> dict:
"""Backfill all organization source extensions from legacy parser tables."""
result = OrganizationSourceBackfillService.backfill()
invalidate_organization_api_cache()
payload = {"batch_size": batch_size, **asdict(result)}
logger.info("All organization source extensions backfilled: %s", payload)
return payload
@shared_task
def backfill_organization_sources_for_parser_batch(
*,
source: str,
batch_id: int,
batch_size: int = 100,
) -> dict:
"""Backfill source extensions affected by one parser batch."""
result = OrganizationSourceBackfillService.backfill(
source=source,
batch_id=batch_id,
)
invalidate_organization_api_cache()
payload = {
"source": source,
"batch_id": batch_id,
"batch_size": batch_size,
**asdict(result),
}
logger.info("Organization source extensions backfilled: %s", payload)
return payload
@shared_task
def refresh_all_organization_data_snapshots(batch_size: int = 100) -> dict:
"""Deprecated compatibility wrapper for the old snapshot schedule."""
return backfill_all_organization_sources(batch_size=batch_size)
@shared_task
def refresh_organization_data_snapshots_for_parser_batch(
*,
source: str,
batch_id: int,
batch_size: int = 100,
) -> dict:
"""Deprecated compatibility wrapper for old parser task imports."""
return backfill_organization_sources_for_parser_batch(
source=source,
batch_id=batch_id,
batch_size=batch_size,
)