feat(registries): add SME and budget imports and fix source API workflows
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 9m37s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 4m18s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m48s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 9m37s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 4m18s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m48s
This commit is contained in:
138
src/apps/parsers/tasks_registry_snapshots.py
Normal file
138
src/apps/parsers/tasks_registry_snapshots.py
Normal file
@@ -0,0 +1,138 @@
|
||||
"""Celery entrypoints for the two public organization registry snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
|
||||
from apps.parsers.models import ParserLoadLog
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from apps.parsers.services import ParserLoadLogService
|
||||
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
|
||||
from celery import shared_task
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||||
"""Run inside the publication transaction, alongside the new records."""
|
||||
job = BackgroundJob.objects.select_for_update().get(pk=job.pk)
|
||||
if job.status != JobStatus.STARTED:
|
||||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.SUCCESS,
|
||||
records_count=result.published,
|
||||
)
|
||||
payload = {
|
||||
"status": "success",
|
||||
"batch_id": load_log.batch_id,
|
||||
"load_id": load_log.id,
|
||||
"artifact_id": str(artifact.uid),
|
||||
"raw_records_count": result.parsed,
|
||||
"published_records_count": result.published,
|
||||
"quarantine_records_count": result.quarantined,
|
||||
"skipped_records_count": result.skipped,
|
||||
}
|
||||
job.update_progress(100, "Снимок реестра опубликован")
|
||||
job.complete(result=payload)
|
||||
return payload
|
||||
|
||||
|
||||
def _run_snapshot(
|
||||
task, *, source: str, refresh: Callable, requested_by_id: int | None
|
||||
) -> dict:
|
||||
job = None
|
||||
load_log = None
|
||||
try:
|
||||
job, _ = BackgroundJob.objects.get_or_create(
|
||||
task_id=task.request.id or str(uuid.uuid4()),
|
||||
defaults={
|
||||
"task_name": task.name,
|
||||
"user_id": requested_by_id,
|
||||
"meta": {"source": source, "source_key": source},
|
||||
},
|
||||
)
|
||||
claimed = BackgroundJob.objects.filter(
|
||||
pk=job.pk,
|
||||
status__in=(JobStatus.PENDING, JobStatus.RETRY),
|
||||
).update(
|
||||
status=JobStatus.STARTED,
|
||||
started_at=timezone.now(),
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
if not claimed:
|
||||
job.refresh_from_db()
|
||||
return job.result or {"status": job.status, "duplicate_delivery": True}
|
||||
job.refresh_from_db()
|
||||
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
||||
source=source,
|
||||
status=ParserLoadLog.Status.IN_PROGRESS,
|
||||
)
|
||||
metadata = {
|
||||
**job.meta,
|
||||
"source": source,
|
||||
"source_key": source,
|
||||
"batch_id": batch_id,
|
||||
}
|
||||
attached = BackgroundJob.objects.filter(
|
||||
pk=job.pk, status=JobStatus.STARTED
|
||||
).update(meta=metadata, updated_at=timezone.now())
|
||||
if not attached:
|
||||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||||
job.refresh_from_db()
|
||||
job.update_progress(5, "Загрузка и проверка снимка реестра")
|
||||
payload = {}
|
||||
|
||||
def finalize(artifact, result):
|
||||
payload.update(_complete_snapshot_job(job, load_log, artifact, result))
|
||||
|
||||
refresh(
|
||||
load_batch=batch_id, uploaded_by_id=requested_by_id, on_publish=finalize
|
||||
)
|
||||
return payload
|
||||
except Exception as exc:
|
||||
if job is not None:
|
||||
job.refresh_from_db()
|
||||
if job.status == JobStatus.SUCCESS:
|
||||
return job.result
|
||||
# Store a bounded error class, not HTTP request internals or raw source data.
|
||||
message = f"Не удалось обновить реестр ({type(exc).__name__})"
|
||||
if isinstance(exc, SnapshotValidationError):
|
||||
message += f": {str(exc)[:200]}"
|
||||
if load_log is not None:
|
||||
ParserLoadLogService.mark_failed(load_log, message)
|
||||
if job is not None:
|
||||
job.fail(error=message)
|
||||
raise
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
name="parsers.fns_sme_support_recipients.refresh",
|
||||
soft_time_limit=4 * 3600,
|
||||
time_limit=4 * 3600 + 300,
|
||||
)
|
||||
def parse_sme_support_recipients(self, *, requested_by_id: int | None = None) -> dict:
|
||||
return _run_snapshot(
|
||||
self,
|
||||
source=SME_SOURCE,
|
||||
refresh=refresh_sme_support,
|
||||
requested_by_id=requested_by_id,
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
name="parsers.budget_ubpandnubp.refresh",
|
||||
soft_time_limit=4 * 3600,
|
||||
time_limit=4 * 3600 + 300,
|
||||
)
|
||||
def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict:
|
||||
return _run_snapshot(
|
||||
self,
|
||||
source=BUDGET_SOURCE,
|
||||
refresh=refresh_budget_registry,
|
||||
requested_by_id=requested_by_id,
|
||||
)
|
||||
Reference in New Issue
Block a user