"""Celery entrypoints for the organization registry snapshots.""" from __future__ import annotations import logging import uuid from collections.abc import Callable from functools import partial from apps.core.exceptions import ConflictError from apps.core.models import BackgroundJob, JobStatus from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry from apps.parsers.models import ParserBatchSequence, ParserLoadLog, ParserSourceArtifact from apps.parsers.registry_snapshots import SnapshotValidationError from apps.parsers.services import ParserLoadLogService from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support from celery import shared_task from django.db import transaction from django.utils import timezone logger = logging.getLogger(__name__) def _refresh_published_exports() -> None: from organizations.tasks import refresh_source_record_export_artifacts try: refresh_source_record_export_artifacts.delay() except Exception as exc: # Publication already committed; export readiness remains separately visible. logger.error("Published source export enqueue failed (%s)", type(exc).__name__) def _complete_snapshot_job(job, load_log, artifact, result) -> dict: """Run inside the publication transaction, alongside the new records.""" job = BackgroundJob.objects.select_for_update().get(pk=job.pk) if job.status != JobStatus.STARTED: raise SnapshotValidationError("snapshot_job_no_longer_active") ParserLoadLogService.update( load_log, status=ParserLoadLog.Status.SUCCESS, records_count=result.published, ) payload = { "status": "success", "batch_id": load_log.batch_id, "load_id": load_log.id, "artifact_id": str(artifact.uid), "raw_records_count": result.parsed, "published_records_count": result.published, "records_count": result.published, "organizations_count": artifact.metadata.get("snapshot_organizations_count", 0), "quarantine_records_count": result.quarantined, "skipped_records_count": result.skipped, "source": artifact.source, "snapshot_date": str(artifact.source_published_at) if artifact.source_published_at else None, "source_published_at": str(artifact.source_published_at) if artifact.source_published_at else None, "version": artifact.version or None, "checksum_sha256": artifact.sha256 or None, } if artifact.source in ( "sro_membership_check", "fssp_enforcement_proceedings", "stock_exchange_activity", "checko_leasing", "checko_messages", "checko_licenses", ): payload.update(artifact.metadata) job.meta = { **job.meta, **artifact.metadata, "published_records_count": result.published, "quarantined_records_count": result.quarantined, } job.save(update_fields=("meta", "updated_at")) job.update_progress(100, "Снимок реестра опубликован") job.complete(result=payload) transaction.on_commit(_refresh_published_exports) return payload def _claim_snapshot_job(job, source, task_name): with transaction.atomic(): if source in ( "sro_membership_check", "fedresurs_company_profile", "fssp_enforcement_proceedings", "stock_exchange_activity", "checko_leasing", "checko_messages", "checko_licenses", ): sequence, _ = ParserBatchSequence.objects.get_or_create(source=source) ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk) if ( BackgroundJob.objects.filter( task_name=task_name, status=JobStatus.STARTED ) .exclude(pk=job.pk) .exists() ): raise ConflictError( code="refresh_already_running", message="Источник уже обновляется" ) claimed = BackgroundJob.objects.filter( pk=job.pk, status__in=(JobStatus.PENDING, JobStatus.RETRY), ).update( status=JobStatus.STARTED, started_at=timezone.now(), updated_at=timezone.now(), ) return claimed def _update_snapshot_progress(job, queried, candidates, found, quarantined): job.refresh_from_db() if job.status != JobStatus.STARTED: raise SnapshotValidationError("snapshot_job_no_longer_active") if job.meta.get("source") == "fssp_enforcement_proceedings": job.update_progress(5, f"Прочитано {queried} строк CSV") BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update( meta={ **job.meta, "raw_records_count": queried, "matched_records_count": found, "quarantined_records_count": quarantined, }, updated_at=timezone.now(), ) return if queried % 10 == 0 or queried == candidates: job.update_progress( 5 + (90 * queried // max(candidates, 1)), f"Проверено {queried} из {candidates} организаций", ) BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update( meta={ **job.meta, "candidate_organizations_count": candidates, "queried_organizations_count": queried, "found_memberships_count": found, "published_records_count": 0, "quarantined_records_count": quarantined, }, updated_at=timezone.now(), ) def _snapshot_error_message(exc: Exception) -> str: message = f"Не удалось обновить реестр ({type(exc).__name__})" if isinstance(exc, SnapshotValidationError): message += f": {str(exc)[:200]}" if isinstance(exc, ConflictError) and exc.code == "upstream_access_not_approved": message += ": upstream_access_not_approved" return message def _attach_rejected_section(job, source, load_log): if ( source in ( "stock_exchange_activity", "checko_leasing", "checko_messages", "checko_licenses", ) and load_log is not None ): rejected = ( ParserSourceArtifact.objects.filter( source=source, load_batch=load_log.batch_id, status=ParserSourceArtifact.Status.REJECTED, metadata__dataset__isnull=True, ) .order_by("-created_at") .first() ) if rejected is not None: job.result = { **rejected.metadata, "source": source, "artifact_id": str(rejected.uid), "batch_id": load_log.batch_id, "status": "failure", } job.meta = {**job.meta, **rejected.metadata} job.save(update_fields=["result", "meta", "updated_at"]) def _run_snapshot( task, *, source: str, refresh: Callable, requested_by_id: int | None, track_progress: bool = False, mode: str | None = None, ) -> dict: job = None load_log = None try: job, _ = BackgroundJob.objects.get_or_create( task_id=task.request.id or str(uuid.uuid4()), defaults={ "task_name": task.name, "user_id": requested_by_id, "meta": {"source": source, "source_key": source}, }, ) claimed = _claim_snapshot_job(job, source, task.name) if not claimed: job.refresh_from_db() return job.result or {"status": job.status, "duplicate_delivery": True} job.refresh_from_db() load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=source, status=ParserLoadLog.Status.IN_PROGRESS, ) metadata = { **job.meta, "source": source, "source_key": source, "batch_id": batch_id, } if mode is not None: metadata["mode"] = mode attached = BackgroundJob.objects.filter( pk=job.pk, status=JobStatus.STARTED ).update(meta=metadata, updated_at=timezone.now()) if not attached: raise SnapshotValidationError("snapshot_job_no_longer_active") job.refresh_from_db() job.update_progress(5, "Загрузка и проверка снимка реестра") payload = {} def finalize(artifact, result): payload.update(_complete_snapshot_job(job, load_log, artifact, result)) kwargs = ( {"on_progress": partial(_update_snapshot_progress, job)} if track_progress else {} ) refresh( load_batch=batch_id, uploaded_by_id=requested_by_id, on_publish=finalize, **kwargs, ) return payload except Exception as exc: if job is not None: job.refresh_from_db() if job.status == JobStatus.SUCCESS: return job.result # Store a bounded error class, not HTTP request internals or raw source data. message = _snapshot_error_message(exc) if load_log is not None: ParserLoadLogService.mark_failed(load_log, message) if job is not None: _attach_rejected_section(job, source, load_log) job.fail(error=message) raise @shared_task( bind=True, name="parsers.fns_sme_support_recipients.refresh", soft_time_limit=4 * 3600, time_limit=4 * 3600 + 300, ) def parse_sme_support_recipients(self, *, requested_by_id: int | None = None) -> dict: return _run_snapshot( self, source=SME_SOURCE, refresh=refresh_sme_support, requested_by_id=requested_by_id, ) @shared_task( bind=True, name="parsers.budget_ubpandnubp.refresh", soft_time_limit=4 * 3600, time_limit=4 * 3600 + 300, ) def parse_budget_registry(self, *, requested_by_id: int | None = None) -> dict: return _run_snapshot( self, source=BUDGET_SOURCE, refresh=refresh_budget_registry, requested_by_id=requested_by_id, ) @shared_task( bind=True, name="parsers.sro_membership_check.refresh", soft_time_limit=12 * 3600, time_limit=12 * 3600 + 300, ) def parse_sro_membership( self, *, requested_by_id: int | None = None, mode: str = "incremental" ) -> dict: from apps.parsers.sro_membership import refresh_sro_membership # Direct Celery delivery cannot bypass the same gate used by HTTP admission. # refresh_sro_membership checks before creating an artifact or doing IO; # _run_snapshot also finalizes any precreated job when approval was revoked. return _run_snapshot( self, source="sro_membership_check", refresh=partial(refresh_sro_membership, mode=mode), requested_by_id=requested_by_id, track_progress=True, mode=mode, ) @shared_task( bind=True, name="parsers.fedresurs_company_profile.refresh", soft_time_limit=24 * 3600, time_limit=24 * 3600 + 300, ) def parse_fedresurs_company_profiles( self, *, requested_by_id: int | None = None, mode: str = "full", organization_uids: list[str] | None = None, ) -> dict: from apps.parsers.fedresurs_snapshot import refresh_fedresurs_company_profiles return _run_snapshot( self, source="fedresurs_company_profile", refresh=partial( refresh_fedresurs_company_profiles, mode=mode, organization_uids=organization_uids, ), requested_by_id=requested_by_id, track_progress=True, mode=mode, ) @shared_task( bind=True, name="parsers.fssp_enforcement_proceedings.refresh", soft_time_limit=8 * 3600, time_limit=8 * 3600 + 300, ) def parse_fssp_enforcement_proceedings( self, *, requested_by_id: int | None = None ) -> dict: from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings return _run_snapshot( self, source="fssp_enforcement_proceedings", refresh=refresh_fssp_enforcement_proceedings, requested_by_id=requested_by_id, track_progress=True, ) @shared_task( bind=True, name="parsers.checko_leasing.refresh", soft_time_limit=8 * 3600, time_limit=8 * 3600 + 300, ) def parse_checko_leasing( self, *, requested_by_id: int | None = None, mode: str = "collection" ) -> dict: if mode == "details": return _run_leasing_details(self, requested_by_id=requested_by_id) if mode != "collection": raise SnapshotValidationError("invalid_mode") from apps.parsers.checko_section_snapshots import refresh_checko_section return _run_snapshot( self, source="checko_leasing", refresh=partial(refresh_checko_section, source="checko_leasing"), requested_by_id=requested_by_id, track_progress=True, ) def _run_leasing_details(task, *, requested_by_id: int | None) -> dict: """Detail-only job; never create a collection log or advance full snapshot success.""" from apps.parsers.leasing_details import enrich_leasing_details job, _ = BackgroundJob.objects.get_or_create( task_id=task.request.id or str(uuid.uuid4()), defaults={ "task_name": task.name, "user_id": requested_by_id, "meta": {"source": "checko_leasing", "mode": "details"}, }, ) try: if not _claim_snapshot_job(job, "checko_leasing", task.name): job.refresh_from_db() return job.result or {"status": job.status, "duplicate_delivery": True} job.update_progress(5, "Дополнение сохранённых событий лизинга") result = enrich_leasing_details() job.refresh_from_db() if job.status != JobStatus.STARTED: raise SnapshotValidationError("snapshot_job_no_longer_active") job.meta = {**job.meta, "mode": "details", **result} job.result = {"source": "checko_leasing", "mode": "details", **result} job.save(update_fields=("meta", "result", "updated_at")) if result["error_code"] and not result["records_updated"]: raise SnapshotValidationError(result["error_code"]) status = "partial" if result["error_code"] else "success" job.update_progress(100, "Порция дополнения лизинга завершена") job.complete(result={"status": status, **job.result}) if result["records_updated"]: transaction.on_commit(_refresh_published_exports) return job.result except Exception as exc: job.fail(error=_snapshot_error_message(exc)) raise @shared_task( bind=True, name="parsers.checko_messages.refresh", soft_time_limit=8 * 3600, time_limit=8 * 3600 + 300, ) def parse_checko_messages(self, *, requested_by_id: int | None = None) -> dict: from apps.parsers.checko_section_snapshots import refresh_checko_section return _run_snapshot( self, source="checko_messages", refresh=partial(refresh_checko_section, source="checko_messages"), requested_by_id=requested_by_id, track_progress=True, ) @shared_task( bind=True, name="parsers.checko_licenses.refresh", soft_time_limit=8 * 3600, time_limit=8 * 3600 + 300, ) def parse_checko_licenses(self, *, requested_by_id: int | None = None) -> dict: from apps.parsers.checko_section_snapshots import refresh_checko_section return _run_snapshot( self, source="checko_licenses", refresh=partial(refresh_checko_section, source="checko_licenses"), requested_by_id=requested_by_id, track_progress=True, ) @shared_task( bind=True, name="parsers.stock_exchange_activity.refresh", soft_time_limit=8 * 3600, time_limit=8 * 3600 + 300, ) def parse_stock_exchange_activity(self, *, requested_by_id: int | None = None) -> dict: from apps.parsers.exchange_snapshot import refresh_stock_exchange_activity return _run_snapshot( self, source="stock_exchange_activity", refresh=refresh_stock_exchange_activity, requested_by_id=requested_by_id, track_progress=True, )