feat: add separate all-years source export task

This commit is contained in:
Aleksandr Meshchryakov
2026-09-28 12:06:09 +02:00
parent 245edd1fff
commit cd5a1764a8
5 changed files with 187 additions and 13 deletions

View File

@@ -22,6 +22,16 @@ logger = logging.getLogger(__name__)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
"""Build and atomically publish the nightly source-record export matrix."""
return _refresh_source_record_export_artifacts(self, all_years=False)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_all_years_source_record_export_artifacts(self) -> dict:
"""Publish all available years independently of the annual generation."""
return _refresh_source_record_export_artifacts(self, all_years=True)
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
lock_key = getattr(
settings,
"SOURCE_RECORD_EXPORT_LOCK_KEY",
@@ -37,7 +47,7 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
countdown = 30
max_retries = max(1, math.ceil(lock_ttl / countdown) + 2)
try:
raise self.retry(countdown=countdown, max_retries=max_retries)
raise task.retry(countdown=countdown, max_retries=max_retries)
except MaxRetriesExceededError:
logger.error(
"Source-record export retry budget exhausted while waiting for build lock"
@@ -45,7 +55,7 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
raise
try:
generation = build_source_record_export_artifacts()
generation = build_source_record_export_artifacts(all_years=all_years)
result = {
"status": "success",
"generation_id": generation.generation_id,