Files
mostovik-backend/src/apps/parsers/tasks_registry_snapshots.py
Aleksandr Meshchryakov ccb7f45c7e
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s
feat: enrich saved leasing events with bounded Fedresurs details
2026-10-06 14:43:21 +02:00

489 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Celery entrypoints for the organization registry snapshots."""
from __future__ import annotations
import logging
import uuid
from collections.abc import Callable
from functools import partial
from apps.core.exceptions import ConflictError
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
from apps.parsers.models import ParserBatchSequence, ParserLoadLog, ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.services import ParserLoadLogService
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
from celery import shared_task
from django.db import transaction
from django.utils import timezone
logger = logging.getLogger(__name__)
def _refresh_published_exports() -> None:
from organizations.tasks import refresh_source_record_export_artifacts
try:
refresh_source_record_export_artifacts.delay()
except Exception as exc:
# Publication already committed; export readiness remains separately visible.
logger.error("Published source export enqueue failed (%s)", type(exc).__name__)
def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"""Run inside the publication transaction, alongside the new records."""
job = BackgroundJob.objects.select_for_update().get(pk=job.pk)
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.SUCCESS,
records_count=result.published,
)
payload = {
"status": "success",
"batch_id": load_log.batch_id,
"load_id": load_log.id,
"artifact_id": str(artifact.uid),
"raw_records_count": result.parsed,
"published_records_count": result.published,
"records_count": result.published,
"organizations_count": artifact.metadata.get("snapshot_organizations_count", 0),
"quarantine_records_count": result.quarantined,
"skipped_records_count": result.skipped,
"source": artifact.source,
"snapshot_date": str(artifact.source_published_at)
if artifact.source_published_at
else None,
"source_published_at": str(artifact.source_published_at)
if artifact.source_published_at
else None,
"version": artifact.version or None,
"checksum_sha256": artifact.sha256 or None,
}
if artifact.source in (
"sro_membership_check",
"fssp_enforcement_proceedings",
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
):
payload.update(artifact.metadata)
job.meta = {
**job.meta,
**artifact.metadata,
"published_records_count": result.published,
"quarantined_records_count": result.quarantined,
}
job.save(update_fields=("meta", "updated_at"))
job.update_progress(100, "Снимок реестра опубликован")
job.complete(result=payload)
transaction.on_commit(_refresh_published_exports)
return payload
def _claim_snapshot_job(job, source, task_name):
with transaction.atomic():
if source in (
"sro_membership_check",
"fedresurs_company_profile",
"fssp_enforcement_proceedings",
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
):
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
if (
BackgroundJob.objects.filter(
task_name=task_name, status=JobStatus.STARTED
)
.exclude(pk=job.pk)
.exists()
):
raise ConflictError(
code="refresh_already_running", message="Источник уже обновляется"
)
claimed = BackgroundJob.objects.filter(
pk=job.pk,
status__in=(JobStatus.PENDING, JobStatus.RETRY),
).update(
status=JobStatus.STARTED,
started_at=timezone.now(),
updated_at=timezone.now(),
)
return claimed
def _update_snapshot_progress(job, queried, candidates, found, quarantined):
job.refresh_from_db()
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
if job.meta.get("source") == "fssp_enforcement_proceedings":
job.update_progress(5, f"Прочитано {queried} строк CSV")
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
meta={
**job.meta,
"raw_records_count": queried,
"matched_records_count": found,
"quarantined_records_count": quarantined,
},
updated_at=timezone.now(),
)
return
if queried % 10 == 0 or queried == candidates:
job.update_progress(
5 + (90 * queried // max(candidates, 1)),
f"Проверено {queried} из {candidates} организаций",
)
BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update(
meta={
**job.meta,
"candidate_organizations_count": candidates,
"queried_organizations_count": queried,
"found_memberships_count": found,
"published_records_count": 0,
"quarantined_records_count": quarantined,
},
updated_at=timezone.now(),
)
def _snapshot_error_message(exc: Exception) -> str:
message = f"Не удалось обновить реестр ({type(exc).__name__})"
if isinstance(exc, SnapshotValidationError):
message += f": {str(exc)[:200]}"
if isinstance(exc, ConflictError) and exc.code == "upstream_access_not_approved":
message += ": upstream_access_not_approved"
return message
def _attach_rejected_section(job, source, load_log):
if (
source
in (
"stock_exchange_activity",
"checko_leasing",
"checko_messages",
"checko_licenses",
)
and load_log is not None
):
rejected = (
ParserSourceArtifact.objects.filter(
source=source,
load_batch=load_log.batch_id,
status=ParserSourceArtifact.Status.REJECTED,
metadata__dataset__isnull=True,
)
.order_by("-created_at")
.first()
)
if rejected is not None:
job.result = {
**rejected.metadata,
"source": source,
"artifact_id": str(rejected.uid),
"batch_id": load_log.batch_id,
"status": "failure",
}
job.meta = {**job.meta, **rejected.metadata}
job.save(update_fields=["result", "meta", "updated_at"])
def _run_snapshot(
task,
*,
source: str,
refresh: Callable,
requested_by_id: int | None,
track_progress: bool = False,
mode: str | None = None,
) -> dict:
job = None
load_log = None
try:
job, _ = BackgroundJob.objects.get_or_create(
task_id=task.request.id or str(uuid.uuid4()),
defaults={
"task_name": task.name,
"user_id": requested_by_id,
"meta": {"source": source, "source_key": source},
},
)
claimed = _claim_snapshot_job(job, source, task.name)
if not claimed:
job.refresh_from_db()
return job.result or {"status": job.status, "duplicate_delivery": True}
job.refresh_from_db()
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
source=source,
status=ParserLoadLog.Status.IN_PROGRESS,
)
metadata = {
**job.meta,
"source": source,
"source_key": source,
"batch_id": batch_id,
}
if mode is not None:
metadata["mode"] = mode
attached = BackgroundJob.objects.filter(
pk=job.pk, status=JobStatus.STARTED
).update(meta=metadata, updated_at=timezone.now())
if not attached:
raise SnapshotValidationError("snapshot_job_no_longer_active")
job.refresh_from_db()
job.update_progress(5, "Загрузка и проверка снимка реестра")
payload = {}
def finalize(artifact, result):
payload.update(_complete_snapshot_job(job, load_log, artifact, result))
kwargs = (
{"on_progress": partial(_update_snapshot_progress, job)}
if track_progress
else {}
)
refresh(
load_batch=batch_id,
uploaded_by_id=requested_by_id,
on_publish=finalize,
**kwargs,
)
return payload
except Exception as exc:
if job is not None:
job.refresh_from_db()
if job.status == JobStatus.SUCCESS:
return job.result
# Store a bounded error class, not HTTP request internals or raw source data.
message = _snapshot_error_message(exc)
if load_log is not None:
ParserLoadLogService.mark_failed(load_log, message)
if job is not None:
_attach_rejected_section(job, source, load_log)
job.fail(error=message)
raise
@shared_task(
bind=True,
name="parsers.fns_sme_support_recipients.refresh",
soft_time_limit=4 * 3600,
time_limit=4 * 3600 + 300,
)
def parse_sme_support_recipients(self, *, requested_by_id: int | None = None) -> dict:
return _run_snapshot(
self,
source=SME_SOURCE,
refresh=refresh_sme_support,
requested_by_id=requested_by_id,
)
@shared_task(
bind=True,
name="parsers.budget_ubpandnubp.refresh",
soft_time_limit=4 * 3600,
time_limit=4 * 3600 + 300,
)
def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict:
return _run_snapshot(
self,
source=BUDGET_SOURCE,
refresh=refresh_budget_registry,
requested_by_id=requested_by_id,
)
@shared_task(
bind=True,
name="parsers.sro_membership_check.refresh",
soft_time_limit=12 * 3600,
time_limit=12 * 3600 + 300,
)
def parse_sro_membership(
self, *, requested_by_id: int | None = None, mode: str = "incremental"
) -> dict:
from apps.parsers.sro_membership import refresh_sro_membership
# Direct Celery delivery cannot bypass the same gate used by HTTP admission.
# refresh_sro_membership checks before creating an artifact or doing IO;
# _run_snapshot also finalizes any precreated job when approval was revoked.
return _run_snapshot(
self,
source="sro_membership_check",
refresh=partial(refresh_sro_membership, mode=mode),
requested_by_id=requested_by_id,
track_progress=True,
mode=mode,
)
@shared_task(
bind=True,
name="parsers.fedresurs_company_profile.refresh",
soft_time_limit=24 * 3600,
time_limit=24 * 3600 + 300,
)
def parse_fedresurs_company_profiles(
self,
*,
requested_by_id: int | None = None,
mode: str = "full",
organization_uids: list[str] | None = None,
) -> dict:
from apps.parsers.fedresurs_snapshot import refresh_fedresurs_company_profiles
return _run_snapshot(
self,
source="fedresurs_company_profile",
refresh=partial(
refresh_fedresurs_company_profiles,
mode=mode,
organization_uids=organization_uids,
),
requested_by_id=requested_by_id,
track_progress=True,
mode=mode,
)
@shared_task(
bind=True,
name="parsers.fssp_enforcement_proceedings.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_fssp_enforcement_proceedings(
self, *, requested_by_id: int | None = None
) -> dict:
from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings
return _run_snapshot(
self,
source="fssp_enforcement_proceedings",
refresh=refresh_fssp_enforcement_proceedings,
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.checko_leasing.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_checko_leasing(
self, *, requested_by_id: int | None = None, mode: str = "collection"
) -> dict:
if mode == "details":
return _run_leasing_details(self, requested_by_id=requested_by_id)
if mode != "collection":
raise SnapshotValidationError("invalid_mode")
from apps.parsers.checko_section_snapshots import refresh_checko_section
return _run_snapshot(
self,
source="checko_leasing",
refresh=partial(refresh_checko_section, source="checko_leasing"),
requested_by_id=requested_by_id,
track_progress=True,
)
def _run_leasing_details(task, *, requested_by_id: int | None) -> dict:
"""Detail-only job; never create a collection log or advance full snapshot success."""
from apps.parsers.leasing_details import enrich_leasing_details
job, _ = BackgroundJob.objects.get_or_create(
task_id=task.request.id or str(uuid.uuid4()),
defaults={
"task_name": task.name,
"user_id": requested_by_id,
"meta": {"source": "checko_leasing", "mode": "details"},
},
)
try:
if not _claim_snapshot_job(job, "checko_leasing", task.name):
job.refresh_from_db()
return job.result or {"status": job.status, "duplicate_delivery": True}
job.update_progress(5, "Дополнение сохранённых событий лизинга")
result = enrich_leasing_details()
job.refresh_from_db()
if job.status != JobStatus.STARTED:
raise SnapshotValidationError("snapshot_job_no_longer_active")
job.meta = {**job.meta, "mode": "details", **result}
job.result = {"source": "checko_leasing", "mode": "details", **result}
job.save(update_fields=("meta", "result", "updated_at"))
if result["error_code"] and not result["records_updated"]:
raise SnapshotValidationError(result["error_code"])
status = "partial" if result["error_code"] else "success"
job.update_progress(100, "Порция дополнения лизинга завершена")
job.complete(result={"status": status, **job.result})
if result["records_updated"]:
transaction.on_commit(_refresh_published_exports)
return job.result
except Exception as exc:
job.fail(error=_snapshot_error_message(exc))
raise
@shared_task(
bind=True,
name="parsers.checko_messages.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_checko_messages(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.checko_section_snapshots import refresh_checko_section
return _run_snapshot(
self,
source="checko_messages",
refresh=partial(refresh_checko_section, source="checko_messages"),
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.checko_licenses.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_checko_licenses(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.checko_section_snapshots import refresh_checko_section
return _run_snapshot(
self,
source="checko_licenses",
refresh=partial(refresh_checko_section, source="checko_licenses"),
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.stock_exchange_activity.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_stock_exchange_activity(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.exchange_snapshot import refresh_stock_exchange_activity
return _run_snapshot(
self,
source="stock_exchange_activity",
refresh=refresh_stock_exchange_activity,
requested_by_id=requested_by_id,
track_progress=True,
)