All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s
489 lines
16 KiB
Python
489 lines
16 KiB
Python
"""Celery entrypoints for the organization registry snapshots."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import uuid
|
||
from collections.abc import Callable
|
||
from functools import partial
|
||
|
||
from apps.core.exceptions import ConflictError
|
||
from apps.core.models import BackgroundJob, JobStatus
|
||
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
|
||
from apps.parsers.models import ParserBatchSequence, ParserLoadLog, ParserSourceArtifact
|
||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||
from apps.parsers.services import ParserLoadLogService
|
||
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
|
||
from celery import shared_task
|
||
from django.db import transaction
|
||
from django.utils import timezone
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def _refresh_published_exports() -> None:
|
||
from organizations.tasks import refresh_source_record_export_artifacts
|
||
|
||
try:
|
||
refresh_source_record_export_artifacts.delay()
|
||
except Exception as exc:
|
||
# Publication already committed; export readiness remains separately visible.
|
||
logger.error("Published source export enqueue failed (%s)", type(exc).__name__)
|
||
|
||
|
||
def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
|
||
"""Run inside the publication transaction, alongside the new records."""
|
||
job = BackgroundJob.objects.select_for_update().get(pk=job.pk)
|
||
if job.status != JobStatus.STARTED:
|
||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||
ParserLoadLogService.update(
|
||
load_log,
|
||
status=ParserLoadLog.Status.SUCCESS,
|
||
records_count=result.published,
|
||
)
|
||
payload = {
|
||
"status": "success",
|
||
"batch_id": load_log.batch_id,
|
||
"load_id": load_log.id,
|
||
"artifact_id": str(artifact.uid),
|
||
"raw_records_count": result.parsed,
|
||
"published_records_count": result.published,
|
||
"records_count": result.published,
|
||
"organizations_count": artifact.metadata.get("snapshot_organizations_count", 0),
|
||
"quarantine_records_count": result.quarantined,
|
||
"skipped_records_count": result.skipped,
|
||
"source": artifact.source,
|
||
"snapshot_date": str(artifact.source_published_at)
|
||
if artifact.source_published_at
|
||
else None,
|
||
"source_published_at": str(artifact.source_published_at)
|
||
if artifact.source_published_at
|
||
else None,
|
||
"version": artifact.version or None,
|
||
"checksum_sha256": artifact.sha256 or None,
|
||
}
|
||
if artifact.source in (
|
||
"sro_membership_check",
|
||
"fssp_enforcement_proceedings",
|
||
"stock_exchange_activity",
|
||
"checko_leasing",
|
||
"checko_messages",
|
||
"checko_licenses",
|
||
):
|
||
payload.update(artifact.metadata)
|
||
job.meta = {
|
||
**job.meta,
|
||
**artifact.metadata,
|
||
"published_records_count": result.published,
|
||
"quarantined_records_count": result.quarantined,
|
||
}
|
||
job.save(update_fields=("meta", "updated_at"))
|
||
job.update_progress(100, "Снимок реестра опубликован")
|
||
job.complete(result=payload)
|
||
transaction.on_commit(_refresh_published_exports)
|
||
return payload
|
||
|
||
|
||
def _claim_snapshot_job(job, source, task_name):
|
||
with transaction.atomic():
|
||
if source in (
|
||
"sro_membership_check",
|
||
"fedresurs_company_profile",
|
||
"fssp_enforcement_proceedings",
|
||
"stock_exchange_activity",
|
||
"checko_leasing",
|
||
"checko_messages",
|
||
"checko_licenses",
|
||
):
|
||
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
|
||
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
|
||
if (
|
||
BackgroundJob.objects.filter(
|
||
task_name=task_name, status=JobStatus.STARTED
|
||
)
|
||
.exclude(pk=job.pk)
|
||
.exists()
|
||
):
|
||
raise ConflictError(
|
||
code="refresh_already_running", message="Источник уже обновляется"
|
||
)
|
||
claimed = BackgroundJob.objects.filter(
|
||
pk=job.pk,
|
||
status__in=(JobStatus.PENDING, JobStatus.RETRY),
|
||
).update(
|
||
status=JobStatus.STARTED,
|
||
started_at=timezone.now(),
|
||
updated_at=timezone.now(),
|
||
)
|
||
return claimed
|
||
|
||
|
||
def _update_snapshot_progress(job, queried, candidates, found, quarantined):
|
||
job.refresh_from_db()
|
||
if job.status != JobStatus.STARTED:
|
||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||
if job.meta.get("source") == "fssp_enforcement_proceedings":
|
||
job.update_progress(5, f"Прочитано {queried} строк CSV")
|
||
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
|
||
meta={
|
||
**job.meta,
|
||
"raw_records_count": queried,
|
||
"matched_records_count": found,
|
||
"quarantined_records_count": quarantined,
|
||
},
|
||
updated_at=timezone.now(),
|
||
)
|
||
return
|
||
if queried % 10 == 0 or queried == candidates:
|
||
job.update_progress(
|
||
5 + (90 * queried // max(candidates, 1)),
|
||
f"Проверено {queried} из {candidates} организаций",
|
||
)
|
||
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
|
||
meta={
|
||
**job.meta,
|
||
"candidate_organizations_count": candidates,
|
||
"queried_organizations_count": queried,
|
||
"found_memberships_count": found,
|
||
"published_records_count": 0,
|
||
"quarantined_records_count": quarantined,
|
||
},
|
||
updated_at=timezone.now(),
|
||
)
|
||
|
||
|
||
def _snapshot_error_message(exc: Exception) -> str:
|
||
message = f"Не удалось обновить реестр ({type(exc).__name__})"
|
||
if isinstance(exc, SnapshotValidationError):
|
||
message += f": {str(exc)[:200]}"
|
||
if isinstance(exc, ConflictError) and exc.code == "upstream_access_not_approved":
|
||
message += ": upstream_access_not_approved"
|
||
return message
|
||
|
||
|
||
def _attach_rejected_section(job, source, load_log):
|
||
if (
|
||
source
|
||
in (
|
||
"stock_exchange_activity",
|
||
"checko_leasing",
|
||
"checko_messages",
|
||
"checko_licenses",
|
||
)
|
||
and load_log is not None
|
||
):
|
||
rejected = (
|
||
ParserSourceArtifact.objects.filter(
|
||
source=source,
|
||
load_batch=load_log.batch_id,
|
||
status=ParserSourceArtifact.Status.REJECTED,
|
||
metadata__dataset__isnull=True,
|
||
)
|
||
.order_by("-created_at")
|
||
.first()
|
||
)
|
||
if rejected is not None:
|
||
job.result = {
|
||
**rejected.metadata,
|
||
"source": source,
|
||
"artifact_id": str(rejected.uid),
|
||
"batch_id": load_log.batch_id,
|
||
"status": "failure",
|
||
}
|
||
job.meta = {**job.meta, **rejected.metadata}
|
||
job.save(update_fields=["result", "meta", "updated_at"])
|
||
|
||
|
||
def _run_snapshot(
|
||
task,
|
||
*,
|
||
source: str,
|
||
refresh: Callable,
|
||
requested_by_id: int | None,
|
||
track_progress: bool = False,
|
||
mode: str | None = None,
|
||
) -> dict:
|
||
job = None
|
||
load_log = None
|
||
try:
|
||
job, _ = BackgroundJob.objects.get_or_create(
|
||
task_id=task.request.id or str(uuid.uuid4()),
|
||
defaults={
|
||
"task_name": task.name,
|
||
"user_id": requested_by_id,
|
||
"meta": {"source": source, "source_key": source},
|
||
},
|
||
)
|
||
claimed = _claim_snapshot_job(job, source, task.name)
|
||
if not claimed:
|
||
job.refresh_from_db()
|
||
return job.result or {"status": job.status, "duplicate_delivery": True}
|
||
job.refresh_from_db()
|
||
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
||
source=source,
|
||
status=ParserLoadLog.Status.IN_PROGRESS,
|
||
)
|
||
metadata = {
|
||
**job.meta,
|
||
"source": source,
|
||
"source_key": source,
|
||
"batch_id": batch_id,
|
||
}
|
||
if mode is not None:
|
||
metadata["mode"] = mode
|
||
attached = BackgroundJob.objects.filter(
|
||
pk=job.pk, status=JobStatus.STARTED
|
||
).update(meta=metadata, updated_at=timezone.now())
|
||
if not attached:
|
||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||
job.refresh_from_db()
|
||
job.update_progress(5, "Загрузка и проверка снимка реестра")
|
||
payload = {}
|
||
|
||
def finalize(artifact, result):
|
||
payload.update(_complete_snapshot_job(job, load_log, artifact, result))
|
||
|
||
kwargs = (
|
||
{"on_progress": partial(_update_snapshot_progress, job)}
|
||
if track_progress
|
||
else {}
|
||
)
|
||
refresh(
|
||
load_batch=batch_id,
|
||
uploaded_by_id=requested_by_id,
|
||
on_publish=finalize,
|
||
**kwargs,
|
||
)
|
||
return payload
|
||
except Exception as exc:
|
||
if job is not None:
|
||
job.refresh_from_db()
|
||
if job.status == JobStatus.SUCCESS:
|
||
return job.result
|
||
# Store a bounded error class, not HTTP request internals or raw source data.
|
||
message = _snapshot_error_message(exc)
|
||
if load_log is not None:
|
||
ParserLoadLogService.mark_failed(load_log, message)
|
||
if job is not None:
|
||
_attach_rejected_section(job, source, load_log)
|
||
job.fail(error=message)
|
||
raise
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.fns_sme_support_recipients.refresh",
|
||
soft_time_limit=4 * 3600,
|
||
time_limit=4 * 3600 + 300,
|
||
)
|
||
def parse_sme_support_recipients(self, *, requested_by_id: int | None = None) -> dict:
|
||
return _run_snapshot(
|
||
self,
|
||
source=SME_SOURCE,
|
||
refresh=refresh_sme_support,
|
||
requested_by_id=requested_by_id,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.budget_ubpandnubp.refresh",
|
||
soft_time_limit=4 * 3600,
|
||
time_limit=4 * 3600 + 300,
|
||
)
|
||
def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict:
|
||
return _run_snapshot(
|
||
self,
|
||
source=BUDGET_SOURCE,
|
||
refresh=refresh_budget_registry,
|
||
requested_by_id=requested_by_id,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.sro_membership_check.refresh",
|
||
soft_time_limit=12 * 3600,
|
||
time_limit=12 * 3600 + 300,
|
||
)
|
||
def parse_sro_membership(
|
||
self, *, requested_by_id: int | None = None, mode: str = "incremental"
|
||
) -> dict:
|
||
from apps.parsers.sro_membership import refresh_sro_membership
|
||
|
||
# Direct Celery delivery cannot bypass the same gate used by HTTP admission.
|
||
# refresh_sro_membership checks before creating an artifact or doing IO;
|
||
# _run_snapshot also finalizes any precreated job when approval was revoked.
|
||
return _run_snapshot(
|
||
self,
|
||
source="sro_membership_check",
|
||
refresh=partial(refresh_sro_membership, mode=mode),
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
mode=mode,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.fedresurs_company_profile.refresh",
|
||
soft_time_limit=24 * 3600,
|
||
time_limit=24 * 3600 + 300,
|
||
)
|
||
def parse_fedresurs_company_profiles(
|
||
self,
|
||
*,
|
||
requested_by_id: int | None = None,
|
||
mode: str = "full",
|
||
organization_uids: list[str] | None = None,
|
||
) -> dict:
|
||
from apps.parsers.fedresurs_snapshot import refresh_fedresurs_company_profiles
|
||
|
||
return _run_snapshot(
|
||
self,
|
||
source="fedresurs_company_profile",
|
||
refresh=partial(
|
||
refresh_fedresurs_company_profiles,
|
||
mode=mode,
|
||
organization_uids=organization_uids,
|
||
),
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
mode=mode,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.fssp_enforcement_proceedings.refresh",
|
||
soft_time_limit=8 * 3600,
|
||
time_limit=8 * 3600 + 300,
|
||
)
|
||
def parse_fssp_enforcement_proceedings(
|
||
self, *, requested_by_id: int | None = None
|
||
) -> dict:
|
||
from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings
|
||
|
||
return _run_snapshot(
|
||
self,
|
||
source="fssp_enforcement_proceedings",
|
||
refresh=refresh_fssp_enforcement_proceedings,
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.checko_leasing.refresh",
|
||
soft_time_limit=8 * 3600,
|
||
time_limit=8 * 3600 + 300,
|
||
)
|
||
def parse_checko_leasing(
|
||
self, *, requested_by_id: int | None = None, mode: str = "collection"
|
||
) -> dict:
|
||
if mode == "details":
|
||
return _run_leasing_details(self, requested_by_id=requested_by_id)
|
||
if mode != "collection":
|
||
raise SnapshotValidationError("invalid_mode")
|
||
from apps.parsers.checko_section_snapshots import refresh_checko_section
|
||
|
||
return _run_snapshot(
|
||
self,
|
||
source="checko_leasing",
|
||
refresh=partial(refresh_checko_section, source="checko_leasing"),
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
)
|
||
|
||
|
||
def _run_leasing_details(task, *, requested_by_id: int | None) -> dict:
|
||
"""Detail-only job; never create a collection log or advance full snapshot success."""
|
||
from apps.parsers.leasing_details import enrich_leasing_details
|
||
|
||
job, _ = BackgroundJob.objects.get_or_create(
|
||
task_id=task.request.id or str(uuid.uuid4()),
|
||
defaults={
|
||
"task_name": task.name,
|
||
"user_id": requested_by_id,
|
||
"meta": {"source": "checko_leasing", "mode": "details"},
|
||
},
|
||
)
|
||
try:
|
||
if not _claim_snapshot_job(job, "checko_leasing", task.name):
|
||
job.refresh_from_db()
|
||
return job.result or {"status": job.status, "duplicate_delivery": True}
|
||
job.update_progress(5, "Дополнение сохранённых событий лизинга")
|
||
result = enrich_leasing_details()
|
||
job.refresh_from_db()
|
||
if job.status != JobStatus.STARTED:
|
||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||
job.meta = {**job.meta, "mode": "details", **result}
|
||
job.result = {"source": "checko_leasing", "mode": "details", **result}
|
||
job.save(update_fields=("meta", "result", "updated_at"))
|
||
if result["error_code"] and not result["records_updated"]:
|
||
raise SnapshotValidationError(result["error_code"])
|
||
status = "partial" if result["error_code"] else "success"
|
||
job.update_progress(100, "Порция дополнения лизинга завершена")
|
||
job.complete(result={"status": status, **job.result})
|
||
if result["records_updated"]:
|
||
transaction.on_commit(_refresh_published_exports)
|
||
return job.result
|
||
except Exception as exc:
|
||
job.fail(error=_snapshot_error_message(exc))
|
||
raise
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.checko_messages.refresh",
|
||
soft_time_limit=8 * 3600,
|
||
time_limit=8 * 3600 + 300,
|
||
)
|
||
def parse_checko_messages(self, *, requested_by_id: int | None = None) -> dict:
|
||
from apps.parsers.checko_section_snapshots import refresh_checko_section
|
||
|
||
return _run_snapshot(
|
||
self,
|
||
source="checko_messages",
|
||
refresh=partial(refresh_checko_section, source="checko_messages"),
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.checko_licenses.refresh",
|
||
soft_time_limit=8 * 3600,
|
||
time_limit=8 * 3600 + 300,
|
||
)
|
||
def parse_checko_licenses(self, *, requested_by_id: int | None = None) -> dict:
|
||
from apps.parsers.checko_section_snapshots import refresh_checko_section
|
||
|
||
return _run_snapshot(
|
||
self,
|
||
source="checko_licenses",
|
||
refresh=partial(refresh_checko_section, source="checko_licenses"),
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
)
|
||
|
||
|
||
@shared_task(
|
||
bind=True,
|
||
name="parsers.stock_exchange_activity.refresh",
|
||
soft_time_limit=8 * 3600,
|
||
time_limit=8 * 3600 + 300,
|
||
)
|
||
def parse_stock_exchange_activity(self, *, requested_by_id: int | None = None) -> dict:
|
||
from apps.parsers.exchange_snapshot import refresh_stock_exchange_activity
|
||
|
||
return _run_snapshot(
|
||
self,
|
||
source="stock_exchange_activity",
|
||
refresh=refresh_stock_exchange_activity,
|
||
requested_by_id=requested_by_id,
|
||
track_progress=True,
|
||
)
|