feat: align source data and nightly exports
This commit is contained in:
@@ -5,14 +5,50 @@ from __future__ import annotations
|
||||
import logging
|
||||
from dataclasses import asdict
|
||||
|
||||
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
||||
from celery import shared_task
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
|
||||
from organizations.cache import invalidate_organization_api_cache
|
||||
from organizations.source_backfill import OrganizationSourceBackfillService
|
||||
from organizations.source_record_export import build_source_record_export_artifacts
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@shared_task(bind=True, base=CorePeriodicTask)
|
||||
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
||||
"""Build and atomically publish the nightly source-record export matrix."""
|
||||
lock_key = getattr(
|
||||
settings,
|
||||
"SOURCE_RECORD_EXPORT_LOCK_KEY",
|
||||
"organizations:source-record-exports:lock",
|
||||
)
|
||||
lock_ttl = int(
|
||||
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
|
||||
)
|
||||
if not cache.add(lock_key, "1", timeout=lock_ttl):
|
||||
logger.info("Source-record export generation skipped: lock is already held")
|
||||
return {"status": "skipped", "reason": "locked"}
|
||||
|
||||
try:
|
||||
generation = build_source_record_export_artifacts()
|
||||
result = {
|
||||
"status": "success",
|
||||
"generation_id": generation.generation_id,
|
||||
"generated_at": generation.generated_at,
|
||||
"artifacts_count": generation.artifacts_count,
|
||||
"files_count": generation.files_count,
|
||||
"records_count": generation.records_count,
|
||||
"total_size": generation.total_size,
|
||||
}
|
||||
logger.info("Source-record export generation published: %s", result)
|
||||
return result
|
||||
finally:
|
||||
cache.delete(lock_key)
|
||||
|
||||
|
||||
@shared_task
|
||||
def backfill_all_organization_sources(batch_size: int = 100) -> dict:
|
||||
"""Backfill all organization source extensions from legacy parser tables."""
|
||||
|
||||
Reference in New Issue
Block a user