feat: complete published registry contracts and gated SRO ingestion
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s

This commit is contained in:
Aleksandr Meshchryakov
2026-09-14 17:01:02 +02:00
parent 49cbfd265c
commit 18971d33ec
76 changed files with 33156 additions and 311 deletions

View File

@@ -1,19 +1,35 @@
"""Celery entrypoints for the two public organization registry snapshots."""
"""Celery entrypoints for the organization registry snapshots."""
from __future__ import annotations
import logging
import uuid
from collections.abc import Callable
from functools import partial
from apps.core.exceptions import ConflictError
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
from apps.parsers.models import ParserLoadLog
from apps.parsers.models import ParserBatchSequence, ParserLoadLog
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.services import ParserLoadLogService
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
from celery import shared_task
from django.db import transaction
from django.utils import timezone
logger = logging.getLogger(__name__)
def _refresh_published_exports() -> None:
from organizations.tasks import refresh_source_record_export_artifacts
try:
refresh_source_record_export_artifacts.delay()
except Exception as exc:
# Publication already committed; export readiness remains separately visible.
logger.error("Published source export enqueue failed (%s)", type(exc).__name__)
def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"""Run inside the publication transaction, alongside the new records."""
@@ -32,16 +48,100 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"artifact_id": str(artifact.uid),
"raw_records_count": result.parsed,
"published_records_count": result.published,
"records_count": result.published,
"organizations_count": artifact.metadata.get("snapshot_organizations_count", 0),
"quarantine_records_count": result.quarantined,
"skipped_records_count": result.skipped,
"source": artifact.source,
"snapshot_date": str(artifact.source_published_at)
if artifact.source_published_at
else None,
"source_published_at": str(artifact.source_published_at)
if artifact.source_published_at
else None,
"version": artifact.version or None,
"checksum_sha256": artifact.sha256 or None,
}
if artifact.source == "sro_membership_check":
payload.update(artifact.metadata)
job.meta = {
**job.meta,
**artifact.metadata,
"published_records_count": result.published,
"quarantined_records_count": result.quarantined,
}
job.save(update_fields=("meta", "updated_at"))
job.update_progress(100, "Снимок реестра опубликован")
job.complete(result=payload)
transaction.on_commit(_refresh_published_exports)
return payload
def _claim_snapshot_job(job, source, task_name):
with transaction.atomic():
if source == "sro_membership_check":
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
if (
BackgroundJob.objects.filter(
task_name=task_name, status=JobStatus.STARTED
)
.exclude(pk=job.pk)
.exists()
):
raise ConflictError(
code="refresh_already_running", message="Источник уже обновляется"
)
claimed = BackgroundJob.objects.filter(
pk=job.pk,
status__in=(JobStatus.PENDING, JobStatus.RETRY),
).update(
status=JobStatus.STARTED,
started_at=timezone.now(),
updated_at=timezone.now(),
)
return claimed
def _update_snapshot_progress(job, queried, candidates, found, quarantined):
job.refresh_from_db()
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
if queried % 10 == 0 or queried == candidates:
job.update_progress(
5 + (90 * queried // max(candidates, 1)),
f"Проверено {queried} из {candidates} организаций",
)
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
meta={
**job.meta,
"candidate_organizations_count": candidates,
"queried_organizations_count": queried,
"found_memberships_count": found,
"published_records_count": 0,
"quarantined_records_count": quarantined,
},
updated_at=timezone.now(),
)
def _snapshot_error_message(exc: Exception) -> str:
message = f"Не удалось обновить реестр ({type(exc).__name__})"
if isinstance(exc, SnapshotValidationError):
message += f": {str(exc)[:200]}"
if isinstance(exc, ConflictError) and exc.code == "upstream_access_not_approved":
message += ": upstream_access_not_approved"
return message
def _run_snapshot(
task, *, source: str, refresh: Callable, requested_by_id: int | None
task,
*,
source: str,
refresh: Callable,
requested_by_id: int | None,
track_progress: bool = False,
mode: str | None = None,
) -> dict:
job = None
load_log = None
@@ -54,14 +154,7 @@ def _run_snapshot(
"meta": {"source": source, "source_key": source},
},
)
claimed = BackgroundJob.objects.filter(
pk=job.pk,
status__in=(JobStatus.PENDING, JobStatus.RETRY),
).update(
status=JobStatus.STARTED,
started_at=timezone.now(),
updated_at=timezone.now(),
)
claimed = _claim_snapshot_job(job, source, task.name)
if not claimed:
job.refresh_from_db()
return job.result or {"status": job.status, "duplicate_delivery": True}
@@ -76,6 +169,8 @@ def _run_snapshot(
"source_key": source,
"batch_id": batch_id,
}
if mode is not None:
metadata["mode"] = mode
attached = BackgroundJob.objects.filter(
pk=job.pk, status=JobStatus.STARTED
).update(meta=metadata, updated_at=timezone.now())
@@ -88,8 +183,16 @@ def _run_snapshot(
def finalize(artifact, result):
payload.update(_complete_snapshot_job(job, load_log, artifact, result))
kwargs = (
{"on_progress": partial(_update_snapshot_progress, job)}
if track_progress
else {}
)
refresh(
load_batch=batch_id, uploaded_by_id=requested_by_id, on_publish=finalize
load_batch=batch_id,
uploaded_by_id=requested_by_id,
on_publish=finalize,
**kwargs,
)
return payload
except Exception as exc:
@@ -98,9 +201,7 @@ def _run_snapshot(
if job.status == JobStatus.SUCCESS:
return job.result
# Store a bounded error class, not HTTP request internals or raw source data.
message = f"Не удалось обновить реестр ({type(exc).__name__})"
if isinstance(exc, SnapshotValidationError):
message += f": {str(exc)[:200]}"
message = _snapshot_error_message(exc)
if load_log is not None:
ParserLoadLogService.mark_failed(load_log, message)
if job is not None:
@@ -136,3 +237,27 @@ def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict:
refresh=refresh_budget_registry,
requested_by_id=requested_by_id,
)
@shared_task(
bind=True,
name="parsers.sro_membership_check.refresh",
soft_time_limit=12 * 3600,
time_limit=12 * 3600 + 300,
)
def parse_sro_membership(
self, *, requested_by_id: int | None = None, mode: str = "incremental"
) -> dict:
from apps.parsers.sro_membership import refresh_sro_membership
# Direct Celery delivery cannot bypass the same gate used by HTTP admission.
# refresh_sro_membership checks before creating an artifact or doing IO;
# _run_snapshot also finalizes any precreated job when approval was revoked.
return _run_snapshot(
self,
source="sro_membership_check",
refresh=partial(refresh_sro_membership, mode=mode),
requested_by_id=requested_by_id,
track_progress=True,
mode=mode,
)