feat(registries): add SME and budget imports and fix source API workflows
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 9m37s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 4m18s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m48s

This commit is contained in:
Aleksandr Meshchryakov
2026-09-13 23:13:47 +02:00
parent c863563851
commit 49cbfd265c
50 changed files with 5021 additions and 452 deletions

View File

@@ -0,0 +1,138 @@
"""Celery entrypoints for the two public organization registry snapshots."""
from __future__ import annotations
import uuid
from collections.abc import Callable
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
from apps.parsers.models import ParserLoadLog
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.services import ParserLoadLogService
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
from celery import shared_task
from django.utils import timezone
def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"""Run inside the publication transaction, alongside the new records."""
job = BackgroundJob.objects.select_for_update().get(pk=job.pk)
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.SUCCESS,
records_count=result.published,
)
payload = {
"status": "success",
"batch_id": load_log.batch_id,
"load_id": load_log.id,
"artifact_id": str(artifact.uid),
"raw_records_count": result.parsed,
"published_records_count": result.published,
"quarantine_records_count": result.quarantined,
"skipped_records_count": result.skipped,
}
job.update_progress(100, "Снимок реестра опубликован")
job.complete(result=payload)
return payload
def _run_snapshot(
task, *, source: str, refresh: Callable, requested_by_id: int | None
) -> dict:
job = None
load_log = None
try:
job, _ = BackgroundJob.objects.get_or_create(
task_id=task.request.id or str(uuid.uuid4()),
defaults={
"task_name": task.name,
"user_id": requested_by_id,
"meta": {"source": source, "source_key": source},
},
)
claimed = BackgroundJob.objects.filter(
pk=job.pk,
status__in=(JobStatus.PENDING, JobStatus.RETRY),
).update(
status=JobStatus.STARTED,
started_at=timezone.now(),
updated_at=timezone.now(),
)
if not claimed:
job.refresh_from_db()
return job.result or {"status": job.status, "duplicate_delivery": True}
job.refresh_from_db()
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
source=source,
status=ParserLoadLog.Status.IN_PROGRESS,
)
metadata = {
**job.meta,
"source": source,
"source_key": source,
"batch_id": batch_id,
}
attached = BackgroundJob.objects.filter(
pk=job.pk, status=JobStatus.STARTED
).update(meta=metadata, updated_at=timezone.now())
if not attached:
raise SnapshotValidationError("snapshot_job_no_longer_active")
job.refresh_from_db()
job.update_progress(5, "Загрузка и проверка снимка реестра")
payload = {}
def finalize(artifact, result):
payload.update(_complete_snapshot_job(job, load_log, artifact, result))
refresh(
load_batch=batch_id, uploaded_by_id=requested_by_id, on_publish=finalize
)
return payload
except Exception as exc:
if job is not None:
job.refresh_from_db()
if job.status == JobStatus.SUCCESS:
return job.result
# Store a bounded error class, not HTTP request internals or raw source data.
message = f"Не удалось обновить реестр ({type(exc).__name__})"
if isinstance(exc, SnapshotValidationError):
message += f": {str(exc)[:200]}"
if load_log is not None:
ParserLoadLogService.mark_failed(load_log, message)
if job is not None:
job.fail(error=message)
raise
@shared_task(
bind=True,
name="parsers.fns_sme_support_recipients.refresh",
soft_time_limit=4 * 3600,
time_limit=4 * 3600 + 300,
)
def parse_sme_support_recipients(self, *, requested_by_id: int | None = None) -> dict:
return _run_snapshot(
self,
source=SME_SOURCE,
refresh=refresh_sme_support,
requested_by_id=requested_by_id,
)
@shared_task(
bind=True,
name="parsers.budget_ubpandnubp.refresh",
soft_time_limit=4 * 3600,
time_limit=4 * 3600 + 300,
)
def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict:
return _run_snapshot(
self,
source=BUDGET_SOURCE,
refresh=refresh_budget_registry,
requested_by_id=requested_by_id,
)