feat: expose Checko section APIs and completion counters

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:45:13 +02:00
parent a517950425
commit 4637b02ddd
26 changed files with 1974 additions and 36 deletions

View File

@@ -0,0 +1,37 @@
"""Three independent monthly collections stay disabled until full dev acceptance."""
from django.db import migrations
SOURCES = ("leasing", "messages", "licenses")
def seed(apps, schema_editor):
intervals = apps.get_model("django_celery_beat", "IntervalSchedule")
tasks = apps.get_model("django_celery_beat", "PeriodicTask")
interval, _ = intervals.objects.get_or_create(every=30, period="days")
for source in SOURCES:
tasks.objects.update_or_create(
name=f"parser:checko-{source}:monthly",
defaults={
"task": f"parsers.checko_{source}.refresh",
"interval": interval,
"enabled": False,
"args": "[]",
"kwargs": "{}",
"description": "Enable only after approved quota/account and full dev acceptance.",
},
)
def unseed(apps, schema_editor):
apps.get_model("django_celery_beat", "PeriodicTask").objects.filter(
name__in=[f"parser:checko-{source}:monthly" for source in SOURCES]
).delete()
class Migration(migrations.Migration):
dependencies = [
("parsers", "0044_checko_sections"),
("django_celery_beat", "0018_improve_crontab_helptext"),
]
operations = [migrations.RunPython(seed, unseed)]

View File

@@ -46,6 +46,9 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60
SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60
SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4
SNAPSHOT_CARD_SLUGS = {
"checko_licenses": "checko-licenses",
"checko_messages": "checko-messages",
"checko_leasing": "checko-leasing",
"fssp_enforcement_proceedings": "fssp-enforcement-proceedings",
"budget_ubpandnubp": "budget-process-registry",
"fns_sme_support_recipients": "sme-support-recipients-registry",
@@ -53,6 +56,9 @@ SNAPSHOT_CARD_SLUGS = {
"fedresurs_company_profile": "fedresurs-company-profile",
}
SNAPSHOT_SOURCE_GROUPS = (
"company_licenses",
"company_messages",
"leasing",
"enforcement_proceedings",
"budget_process_registry",
"government_support",
@@ -505,6 +511,57 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
),
refresh_interval=timedelta(days=1),
),
SourceCardDefinition(
slug="checko-leasing",
title="Лизинг",
description="Лизинг канонических организаций.",
order=170,
task_names=("parsers.checko_leasing.refresh",),
source_items=(
SourceItemDefinition(
code="checko_leasing",
title="Лизинг",
description="Проверенные сведения организаций.",
parser_source=ParserLoadLog.Source.CHECKO_LEASING,
refresh_key="checko_leasing",
),
),
refresh_interval=timedelta(days=30),
),
SourceCardDefinition(
slug="checko-messages",
title="Сообщения организаций",
description="Сообщения канонических организаций.",
order=180,
task_names=("parsers.checko_messages.refresh",),
source_items=(
SourceItemDefinition(
code="checko_messages",
title="Сообщения организаций",
description="Проверенные сведения организаций.",
parser_source=ParserLoadLog.Source.CHECKO_MESSAGES,
refresh_key="checko_messages",
),
),
refresh_interval=timedelta(days=30),
),
SourceCardDefinition(
slug="checko-licenses",
title="Лицензии организаций",
description="Лицензии канонических организаций.",
order=190,
task_names=("parsers.checko_licenses.refresh",),
source_items=(
SourceItemDefinition(
code="checko_licenses",
title="Лицензии организаций",
description="Проверенные сведения организаций.",
parser_source=ParserLoadLog.Source.CHECKO_LICENSES,
refresh_key="checko_licenses",
),
),
refresh_interval=timedelta(days=30),
),
SourceCardDefinition(
slug="fssp-enforcement-proceedings",
title="ФССП: Банк данных исполнительных производств",
@@ -532,6 +589,9 @@ SOURCE_CARD_BY_PARSER_SOURCE = {
if source_item.parser_source
}
GENERIC_RECORD_SOURCES_BY_ITEM_CODE = {
"checko_licenses": ParserLoadLog.Source.CHECKO_LICENSES,
"checko_messages": ParserLoadLog.Source.CHECKO_MESSAGES,
"checko_leasing": ParserLoadLog.Source.CHECKO_LEASING,
"procurements_44fz": ParserLoadLog.Source.PROCUREMENTS_44FZ,
"procurements_223fz": ParserLoadLog.Source.PROCUREMENTS_223FZ,
"contracts": ParserLoadLog.Source.CONTRACTS,
@@ -1030,14 +1090,18 @@ class SourceCardService:
"-updated_at",
"-created_at",
)
for load in queryset:
loads = list(queryset)
attach_snapshot_artifacts(loads)
for load in loads:
latest_load_by_source.setdefault(load.source, load)
if load.status in SUCCESSFUL_LOAD_STATUSES:
artifact = get_snapshot_artifact(load)
partial = (
artifact is not None
and artifact.metadata.get("completeness") == "partial"
)
if load.status in SUCCESSFUL_LOAD_STATUSES and not partial:
latest_success_load_by_source.setdefault(load.source, load)
attach_snapshot_artifacts(
[*latest_load_by_source.values(), *latest_success_load_by_source.values()]
)
return latest_load_by_source, latest_success_load_by_source
@classmethod
@@ -1346,13 +1410,40 @@ class SourceCardService:
elif definition.slug in SNAPSHOT_CARD_SLUGS.values():
from apps.parsers.tasks_registry_snapshots import (
parse_budget_registry,
parse_checko_leasing,
parse_checko_licenses,
parse_checko_messages,
parse_fedresurs_company_profiles,
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,
parse_sro_membership,
)
if definition.slug == "fssp-enforcement-proceedings":
if definition.slug == "checko-leasing":
task_specs = (
(
parse_checko_leasing,
"parsers.checko_leasing.refresh",
"checko_leasing",
),
)
elif definition.slug == "checko-messages":
task_specs = (
(
parse_checko_messages,
"parsers.checko_messages.refresh",
"checko_messages",
),
)
elif definition.slug == "checko-licenses":
task_specs = (
(
parse_checko_licenses,
"parsers.checko_licenses.refresh",
"checko_licenses",
),
)
elif definition.slug == "fssp-enforcement-proceedings":
task_specs = (
(
parse_fssp_enforcement_proceedings,
@@ -1736,7 +1827,17 @@ class SourceCardService:
queryset = ParserLoadLog.objects.filter(source__in=parser_sources)
if statuses:
queryset = queryset.filter(status__in=statuses)
return queryset.order_by("-updated_at", "-created_at").first()
loads = list(queryset.order_by("-updated_at", "-created_at"))
attach_snapshot_artifacts(loads)
for load in loads:
artifact = get_snapshot_artifact(load)
if (
not statuses
or artifact is None
or artifact.metadata.get("completeness") != "partial"
):
return load
return None
@classmethod
def _get_latest_load_by_source(
@@ -1750,7 +1851,17 @@ class SourceCardService:
queryset = ParserLoadLog.objects.filter(source=parser_source)
if statuses:
queryset = queryset.filter(status__in=statuses)
return queryset.order_by("-updated_at", "-created_at").first()
loads = list(queryset.order_by("-updated_at", "-created_at"))
attach_snapshot_artifacts(loads)
for load in loads:
artifact = get_snapshot_artifact(load)
if (
not statuses
or artifact is None
or artifact.metadata.get("completeness") != "partial"
):
return load
return None
@classmethod
def _get_active_tasks(

View File

@@ -56,6 +56,42 @@ class ParserSourceDescriptor:
PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
"checko_licenses": ParserSourceDescriptor(
key="checko_licenses",
source=ParserLoadLog.Source.CHECKO_LICENSES,
title="Лицензии организаций",
agency="Внешний реестр",
data_scope="Лицензии организаций канонических организаций",
task_name="parsers.checko_licenses.refresh",
mode="organization_api",
access_method="monthly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
"checko_messages": ParserSourceDescriptor(
key="checko_messages",
source=ParserLoadLog.Source.CHECKO_MESSAGES,
title="Сообщения организаций",
agency="Внешний реестр",
data_scope="Сообщения организаций канонических организаций",
task_name="parsers.checko_messages.refresh",
mode="organization_api",
access_method="monthly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
"checko_leasing": ParserSourceDescriptor(
key="checko_leasing",
source=ParserLoadLog.Source.CHECKO_LEASING,
title="Лизинг",
agency="Внешний реестр",
data_scope="Лизинг канонических организаций",
task_name="parsers.checko_leasing.refresh",
mode="organization_api",
access_method="monthly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
"fssp_enforcement_proceedings": ParserSourceDescriptor(
key="fssp_enforcement_proceedings",
source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,

View File

@@ -74,6 +74,9 @@ from apps.parsers.source_artifacts import cleanup_parser_source_artifacts
from apps.parsers.source_registry import PARSER_SOURCES
from apps.parsers.tasks_registry_snapshots import ( # noqa: F401
parse_budget_registry,
parse_checko_leasing,
parse_checko_licenses,
parse_checko_messages,
parse_fedresurs_company_profiles,
parse_fssp_enforcement_proceedings,
parse_sme_support_recipients,

View File

@@ -10,7 +10,7 @@ from functools import partial
from apps.core.exceptions import ConflictError
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.budget_registry import BUDGET_SOURCE, refresh_budget_registry
from apps.parsers.models import ParserBatchSequence, ParserLoadLog
from apps.parsers.models import ParserBatchSequence, ParserLoadLog, ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.services import ParserLoadLogService
from apps.parsers.sme_support import SME_SOURCE, refresh_sme_support
@@ -62,7 +62,13 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict:
"version": artifact.version or None,
"checksum_sha256": artifact.sha256 or None,
}
if artifact.source in ("sro_membership_check", "fssp_enforcement_proceedings"):
if artifact.source in (
"sro_membership_check",
"fssp_enforcement_proceedings",
"checko_leasing",
"checko_messages",
"checko_licenses",
):
payload.update(artifact.metadata)
job.meta = {
**job.meta,
@@ -83,6 +89,9 @@ def _claim_snapshot_job(job, source, task_name):
"sro_membership_check",
"fedresurs_company_profile",
"fssp_enforcement_proceedings",
"checko_leasing",
"checko_messages",
"checko_licenses",
):
sequence, _ = ParserBatchSequence.objects.get_or_create(source=source)
ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk)
@@ -150,6 +159,32 @@ def _snapshot_error_message(exc: Exception) -> str:
return message
def _attach_rejected_section(job, source, load_log):
if (
source in ("checko_leasing", "checko_messages", "checko_licenses")
and load_log is not None
):
rejected = (
ParserSourceArtifact.objects.filter(
source=source,
load_batch=load_log.batch_id,
status=ParserSourceArtifact.Status.REJECTED,
)
.order_by("-created_at")
.first()
)
if rejected is not None:
job.result = {
**rejected.metadata,
"source": source,
"artifact_id": str(rejected.uid),
"batch_id": load_log.batch_id,
"status": "failure",
}
job.meta = {**job.meta, **rejected.metadata}
job.save(update_fields=["result", "meta", "updated_at"])
def _run_snapshot(
task,
*,
@@ -221,6 +256,7 @@ def _run_snapshot(
if load_log is not None:
ParserLoadLogService.mark_failed(load_log, message)
if job is not None:
_attach_rejected_section(job, source, load_log)
job.fail(error=message)
raise
@@ -326,3 +362,57 @@ def parse_fssp_enforcement_proceedings(
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.checko_leasing.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_checko_leasing(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.checko_section_snapshots import refresh_checko_section
return _run_snapshot(
self,
source="checko_leasing",
refresh=partial(refresh_checko_section, source="checko_leasing"),
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.checko_messages.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_checko_messages(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.checko_section_snapshots import refresh_checko_section
return _run_snapshot(
self,
source="checko_messages",
refresh=partial(refresh_checko_section, source="checko_messages"),
requested_by_id=requested_by_id,
track_progress=True,
)
@shared_task(
bind=True,
name="parsers.checko_licenses.refresh",
soft_time_limit=8 * 3600,
time_limit=8 * 3600 + 300,
)
def parse_checko_licenses(self, *, requested_by_id: int | None = None) -> dict:
from apps.parsers.checko_section_snapshots import refresh_checko_section
return _run_snapshot(
self,
source="checko_licenses",
refresh=partial(refresh_checko_section, source="checko_licenses"),
requested_by_id=requested_by_id,
track_progress=True,
)

View File

@@ -146,6 +146,9 @@ SOURCE_RESULT_TAGS = {
}
TASKS_BY_NAME = {
"parsers.checko_licenses.refresh": tasks.parse_checko_licenses,
"parsers.checko_messages.refresh": tasks.parse_checko_messages,
"parsers.checko_leasing.refresh": tasks.parse_checko_leasing,
"parsers.fssp_enforcement_proceedings.refresh": tasks.parse_fssp_enforcement_proceedings,
"parsers.fedresurs_company_profile.refresh": tasks.parse_fedresurs_company_profiles,
"apps.parsers.tasks.parse_industrial_production": tasks.parse_industrial_production,

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,41 @@
"""Verified section payloads shared by public list and detail contracts."""
from rest_framework import serializers
from organizations.registry_payload_serializers import RegistryUTCDateTimeField
class CheckoMessagePayloadSerializer(serializers.Serializer):
message_guid = serializers.CharField()
publication_raw = serializers.CharField(allow_null=True)
publication_date = serializers.DateField(allow_null=True)
message_type = serializers.CharField()
message_type_name = serializers.CharField(allow_null=True)
publisher_identifier = serializers.CharField()
participant_identifiers = serializers.ListField(child=serializers.CharField())
source_url = serializers.CharField(allow_null=True)
fetched_at = RegistryUTCDateTimeField()
upstream = serializers.JSONField()
class CheckoLeasingPayloadSerializer(CheckoMessagePayloadSerializer):
lease_event_type = serializers.CharField()
class CheckoLicensePayloadSerializer(serializers.Serializer):
license_number = serializers.CharField()
issued_on = serializers.DateField()
valid_from = serializers.DateField(allow_null=True)
valid_until = serializers.DateField(allow_null=True)
authority = serializers.CharField()
activities = serializers.ListField(child=serializers.CharField())
source_url = serializers.CharField()
fetched_at = RegistryUTCDateTimeField()
upstream = serializers.JSONField()
CHECKO_PAYLOAD_SERIALIZERS = {
"checko_leasing": CheckoLeasingPayloadSerializer,
"checko_messages": CheckoMessagePayloadSerializer,
"checko_licenses": CheckoLicensePayloadSerializer,
}

View File

@@ -322,6 +322,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer):
)
def get_payload(self, obj) -> dict | list | str | int | float | bool | None: # noqa: C901
payload = obj.payload
from organizations.checko_section_payload_serializers import (
CHECKO_PAYLOAD_SERIALIZERS,
)
if obj.source in CHECKO_PAYLOAD_SERIALIZERS:
return CHECKO_PAYLOAD_SERIALIZERS[obj.source](payload).data
if obj.source == "fssp_enforcement_proceedings":
from organizations.fssp_payload_serializers import (
FsspEnforcementProceedingPayloadSerializer,

View File

@@ -49,6 +49,9 @@ EXPORT_FORMAT_JSON = "json"
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
ALL_HISTORY_SOURCE_GROUPS = {
"leasing",
"company_messages",
"company_licenses",
SourceGroup.SRO_MEMBERSHIP.value,
SourceGroup.GOVERNMENT_SUPPORT.value,
SourceGroup.BUDGET_PROCESS_REGISTRY.value,
@@ -80,6 +83,9 @@ EXPORT_PROVIDER_NAME_PATTERN = re.compile(
)
SNAPSHOT_EXPORT_SOURCES = {
"company_licenses": "checko_licenses",
"company_messages": "checko_messages",
"leasing": "checko_leasing",
"enforcement_proceedings": "fssp_enforcement_proceedings",
"budget_process_registry": "budget_ubpandnubp",
"government_support": "fns_sme_support_recipients",
@@ -90,6 +96,9 @@ SNAPSHOT_EXPORT_SOURCES = {
GENERATION_ID_PATTERN = re.compile(r"[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}")
SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = {
"company_licenses": "checko-licenses",
"company_messages": "checko-messages",
"leasing": "checko-leasing",
SourceGroup.ENFORCEMENT_PROCEEDINGS.value: "fssp-enforcement-proceedings",
SourceGroup.SRO_MEMBERSHIP.value: "sro-membership-check",
SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS.value: "integrated-structure-documents",
@@ -1289,7 +1298,11 @@ def _build_record_row(
"updated_at": record.updated_at,
**_flatten_payload(record.payload),
}
public_row = _sanitize_export_provider_value(row)
public_row = (
row
if record.source in ("checko_leasing", "checko_messages", "checko_licenses")
else _sanitize_export_provider_value(row)
)
serialized_row = {
key: _serialize_json_value(value) for key, value in public_row.items()
}

View File

@@ -27,6 +27,9 @@ from drf_yasg import openapi
from rest_framework.exceptions import NotFound
REGISTRY_GROUPS = (
"company_licenses",
"company_messages",
"leasing",
"enforcement_proceedings",
"budget_process_registry",
"government_support",
@@ -35,6 +38,9 @@ REGISTRY_GROUPS = (
"fedresurs_company_profile",
)
REGISTRY_SOURCE_GROUPS = {
"checko_licenses": "company_licenses",
"checko_messages": "company_messages",
"checko_leasing": "leasing",
"fssp_enforcement_proceedings": "enforcement_proceedings",
"budget_ubpandnubp": "budget_process_registry",
"fns_sme_support_recipients": "government_support",
@@ -44,6 +50,12 @@ REGISTRY_SOURCE_GROUPS = {
}
REGISTRY_ORDERING_FIELDS = (
"payload__publication_date",
"payload__message_type",
"payload__license_number",
"payload__issued_on",
"payload__valid_until",
"payload__authority",
"payload__document_raw",
"payload__terminated_on",
"payload__service_text",
@@ -75,6 +87,9 @@ REGISTRY_ORDERING_FIELDS = (
"payload__captured_at",
)
EXACT_FILTERS = {
"message_type": "payload__message_type",
"license_number": "payload__license_number",
"license_authority": "payload__authority",
"organization_type": "payload__classification__organization_type__code",
"establishment_kind": "payload__classification__establishment_kind__code",
"budget_level": "payload__budget__level__code",
@@ -101,6 +116,9 @@ FEDRESURS_SECTION_STATES = (
"schema_unverified",
)
DATE_RANGES = {
"publication_date": "payload__publication_date",
"license_issued": "payload__issued_on",
"license_valid_until": "payload__valid_until",
"date": "canonical_record_date",
"registry_entry": "payload__registry_entry_date",
"support_until": "payload__support_until",

View File

@@ -44,6 +44,9 @@ CANONICAL_ONLY_TEST_SOURCES = {
ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK,
}
FILE_UPLOAD_ONLY_TEST_SOURCES = {
"checko_leasing",
"checko_messages",
"checko_licenses",
# Real FSSP snapshots must not be fabricated by the demo command.
ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
ParserLoadLog.Source.ROPK_SANCTIONS,