feat: complete published registry contracts and gated SRO ingestion
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s
This commit is contained in:
@@ -1,19 +1,35 @@
|
||||
"""Celery entrypoints for the two public organization registry snapshots."""
|
||||
"""Celery entrypoints for the organization registry snapshots."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from functools import partial
|
||||
|
||||
from apps.core.exceptions import ConflictError
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
|
||||
from apps.parsers.models import ParserLoadLog
|
||||
from apps.parsers.models import ParserBatchSequence, ParserLoadLog
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from apps.parsers.services import ParserLoadLogService
|
||||
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
|
||||
from celery import shared_task
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _refresh_published_exports() -> None:
|
||||
from organizations.tasks import refresh_source_record_export_artifacts
|
||||
|
||||
try:
|
||||
refresh_source_record_export_artifacts.delay()
|
||||
except Exception as exc:
|
||||
# Publication already committed; export readiness remains separately visible.
|
||||
logger.error("Published source export enqueue failed (%s)", type(exc).__name__)
|
||||
|
||||
|
||||
def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||||
"""Run inside the publication transaction, alongside the new records."""
|
||||
@@ -32,16 +48,100 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||||
"artifact_id": str(artifact.uid),
|
||||
"raw_records_count": result.parsed,
|
||||
"published_records_count": result.published,
|
||||
"records_count": result.published,
|
||||
"organizations_count": artifact.metadata.get("snapshot_organizations_count", 0),
|
||||
"quarantine_records_count": result.quarantined,
|
||||
"skipped_records_count": result.skipped,
|
||||
"source": artifact.source,
|
||||
"snapshot_date": str(artifact.source_published_at)
|
||||
if artifact.source_published_at
|
||||
else None,
|
||||
"source_published_at": str(artifact.source_published_at)
|
||||
if artifact.source_published_at
|
||||
else None,
|
||||
"version": artifact.version or None,
|
||||
"checksum_sha256": artifact.sha256 or None,
|
||||
}
|
||||
if artifact.source == "sro_membership_check":
|
||||
payload.update(artifact.metadata)
|
||||
job.meta = {
|
||||
**job.meta,
|
||||
**artifact.metadata,
|
||||
"published_records_count": result.published,
|
||||
"quarantined_records_count": result.quarantined,
|
||||
}
|
||||
job.save(update_fields=("meta", "updated_at"))
|
||||
job.update_progress(100, "Снимок реестра опубликован")
|
||||
job.complete(result=payload)
|
||||
transaction.on_commit(_refresh_published_exports)
|
||||
return payload
|
||||
|
||||
|
||||
def _claim_snapshot_job(job, source, task_name):
|
||||
with transaction.atomic():
|
||||
if source == "sro_membership_check":
|
||||
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
|
||||
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
|
||||
if (
|
||||
BackgroundJob.objects.filter(
|
||||
task_name=task_name, status=JobStatus.STARTED
|
||||
)
|
||||
.exclude(pk=job.pk)
|
||||
.exists()
|
||||
):
|
||||
raise ConflictError(
|
||||
code="refresh_already_running", message="Источник уже обновляется"
|
||||
)
|
||||
claimed = BackgroundJob.objects.filter(
|
||||
pk=job.pk,
|
||||
status__in=(JobStatus.PENDING, JobStatus.RETRY),
|
||||
).update(
|
||||
status=JobStatus.STARTED,
|
||||
started_at=timezone.now(),
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
return claimed
|
||||
|
||||
|
||||
def _update_snapshot_progress(job, queried, candidates, found, quarantined):
|
||||
job.refresh_from_db()
|
||||
if job.status != JobStatus.STARTED:
|
||||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||||
if queried % 10 == 0 or queried == candidates:
|
||||
job.update_progress(
|
||||
5 + (90 * queried // max(candidates, 1)),
|
||||
f"Проверено {queried} из {candidates} организаций",
|
||||
)
|
||||
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
|
||||
meta={
|
||||
**job.meta,
|
||||
"candidate_organizations_count": candidates,
|
||||
"queried_organizations_count": queried,
|
||||
"found_memberships_count": found,
|
||||
"published_records_count": 0,
|
||||
"quarantined_records_count": quarantined,
|
||||
},
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
|
||||
|
||||
def _snapshot_error_message(exc: Exception) -> str:
|
||||
message = f"Не удалось обновить реестр ({type(exc).__name__})"
|
||||
if isinstance(exc, SnapshotValidationError):
|
||||
message += f": {str(exc)[:200]}"
|
||||
if isinstance(exc, ConflictError) and exc.code == "upstream_access_not_approved":
|
||||
message += ": upstream_access_not_approved"
|
||||
return message
|
||||
|
||||
|
||||
def _run_snapshot(
|
||||
task, *, source: str, refresh: Callable, requested_by_id: int | None
|
||||
task,
|
||||
*,
|
||||
source: str,
|
||||
refresh: Callable,
|
||||
requested_by_id: int | None,
|
||||
track_progress: bool = False,
|
||||
mode: str | None = None,
|
||||
) -> dict:
|
||||
job = None
|
||||
load_log = None
|
||||
@@ -54,14 +154,7 @@ def _run_snapshot(
|
||||
"meta": {"source": source, "source_key": source},
|
||||
},
|
||||
)
|
||||
claimed = BackgroundJob.objects.filter(
|
||||
pk=job.pk,
|
||||
status__in=(JobStatus.PENDING, JobStatus.RETRY),
|
||||
).update(
|
||||
status=JobStatus.STARTED,
|
||||
started_at=timezone.now(),
|
||||
updated_at=timezone.now(),
|
||||
)
|
||||
claimed = _claim_snapshot_job(job, source, task.name)
|
||||
if not claimed:
|
||||
job.refresh_from_db()
|
||||
return job.result or {"status": job.status, "duplicate_delivery": True}
|
||||
@@ -76,6 +169,8 @@ def _run_snapshot(
|
||||
"source_key": source,
|
||||
"batch_id": batch_id,
|
||||
}
|
||||
if mode is not None:
|
||||
metadata["mode"] = mode
|
||||
attached = BackgroundJob.objects.filter(
|
||||
pk=job.pk, status=JobStatus.STARTED
|
||||
).update(meta=metadata, updated_at=timezone.now())
|
||||
@@ -88,8 +183,16 @@ def _run_snapshot(
|
||||
def finalize(artifact, result):
|
||||
payload.update(_complete_snapshot_job(job, load_log, artifact, result))
|
||||
|
||||
kwargs = (
|
||||
{"on_progress": partial(_update_snapshot_progress, job)}
|
||||
if track_progress
|
||||
else {}
|
||||
)
|
||||
refresh(
|
||||
load_batch=batch_id, uploaded_by_id=requested_by_id, on_publish=finalize
|
||||
load_batch=batch_id,
|
||||
uploaded_by_id=requested_by_id,
|
||||
on_publish=finalize,
|
||||
**kwargs,
|
||||
)
|
||||
return payload
|
||||
except Exception as exc:
|
||||
@@ -98,9 +201,7 @@ def _run_snapshot(
|
||||
if job.status == JobStatus.SUCCESS:
|
||||
return job.result
|
||||
# Store a bounded error class, not HTTP request internals or raw source data.
|
||||
message = f"Не удалось обновить реестр ({type(exc).__name__})"
|
||||
if isinstance(exc, SnapshotValidationError):
|
||||
message += f": {str(exc)[:200]}"
|
||||
message = _snapshot_error_message(exc)
|
||||
if load_log is not None:
|
||||
ParserLoadLogService.mark_failed(load_log, message)
|
||||
if job is not None:
|
||||
@@ -136,3 +237,27 @@ def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict:
|
||||
refresh=refresh_budget_registry,
|
||||
requested_by_id=requested_by_id,
|
||||
)
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
name="parsers.sro_membership_check.refresh",
|
||||
soft_time_limit=12 * 3600,
|
||||
time_limit=12 * 3600 + 300,
|
||||
)
|
||||
def parse_sro_membership(
|
||||
self, *, requested_by_id: int | None = None, mode: str = "incremental"
|
||||
) -> dict:
|
||||
from apps.parsers.sro_membership import refresh_sro_membership
|
||||
|
||||
# Direct Celery delivery cannot bypass the same gate used by HTTP admission.
|
||||
# refresh_sro_membership checks before creating an artifact or doing IO;
|
||||
# _run_snapshot also finalizes any precreated job when approval was revoked.
|
||||
return _run_snapshot(
|
||||
self,
|
||||
source="sro_membership_check",
|
||||
refresh=partial(refresh_sro_membership, mode=mode),
|
||||
requested_by_id=requested_by_id,
|
||||
track_progress=True,
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user