feat: align source data and nightly exports
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 43s
CI/CD Pipeline / Build and Push Images (push) Successful in 19s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 24s

This commit is contained in:
2026-08-03 19:27:07 +02:00
parent dd8697c733
commit b19951d0c9
24 changed files with 1736 additions and 210 deletions

View File

@@ -5,14 +5,50 @@ from __future__ import annotations
import logging
from dataclasses import asdict
from apps.core.tasks import PeriodicTask as CorePeriodicTask
from celery import shared_task
from django.conf import settings
from django.core.cache import cache
from organizations.cache import invalidate_organization_api_cache
from organizations.source_backfill import OrganizationSourceBackfillService
from organizations.source_record_export import build_source_record_export_artifacts
logger = logging.getLogger(__name__)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
"""Build and atomically publish the nightly source-record export matrix."""
lock_key = getattr(
settings,
"SOURCE_RECORD_EXPORT_LOCK_KEY",
"organizations:source-record-exports:lock",
)
lock_ttl = int(
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
)
if not cache.add(lock_key, "1", timeout=lock_ttl):
logger.info("Source-record export generation skipped: lock is already held")
return {"status": "skipped", "reason": "locked"}
try:
generation = build_source_record_export_artifacts()
result = {
"status": "success",
"generation_id": generation.generation_id,
"generated_at": generation.generated_at,
"artifacts_count": generation.artifacts_count,
"files_count": generation.files_count,
"records_count": generation.records_count,
"total_size": generation.total_size,
}
logger.info("Source-record export generation published: %s", result)
return result
finally:
cache.delete(lock_key)
@shared_task
def backfill_all_organization_sources(batch_size: int = 100) -> dict:
"""Backfill all organization source extensions from legacy parser tables."""