feat: expose Checko section APIs and completion counters

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 16:45:13 +02:00
parent a517950425
commit 4637b02ddd
26 changed files with 1974 additions and 36 deletions

View File

@@ -36,7 +36,7 @@ def test_discriminated_records_keep_named_ordering_and_old_operations(contract):
schemas = contract["components"]["schemas"]
for name in ("OrganizationSourceRecord", "OrganizationSourceRecordList"):
record = schemas[name]
assert len(record["oneOf"]) == 7
assert len(record["oneOf"]) == 10
assert record["discriminator"]["propertyName"] == "source_group"
for group, record_type in (
("budget_process_registry", "budget_registry_organization"),

View File

@@ -0,0 +1,208 @@
"""Three independent source APIs and full/partial task semantics."""
from importlib import import_module
from tempfile import TemporaryDirectory
from unittest.mock import patch
import pytest
from apps.core.models import BackgroundJob
from apps.parsers import checko_section_snapshots as snapshots
from apps.parsers.checko_collection import claim_monthly_collection
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.source_cards import SourceCardService
from core.celery import app as celery_app
from django.core.cache import cache
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization
from organizations.source_groups import get_source_group_descriptor
from organizations.source_record_export import _build_record_row, _source_group_queryset
from rest_framework.test import APIClient
from tests.apps.parsers.test_checko_sections import client_with, fixture
from tests.apps.user.factories import UserFactory
SOURCES = {
"checko_leasing": ("leasing", "leasing_contract"),
"checko_messages": ("company_messages", "company_message"),
"checko_licenses": ("company_licenses", "company_license"),
}
class CheckoSectionContractTests(TestCase):
def setUp(self):
tmp = TemporaryDirectory()
self.addCleanup(tmp.cleanup)
settings = override_settings(
MEDIA_ROOT=tmp.name + "/media",
PARSER_PRIVATE_ARTIFACT_ROOT=tmp.name + "/private",
)
settings.enable()
self.addCleanup(settings.disable)
cache.clear()
self.org = Organization.objects.create(
name="АО Проверка",
inn="7707083893",
ogrn="1027700132195",
okpo="12345678",
directory_imported_at=timezone.now(),
gk_code="rosatom",
)
self.client = APIClient()
self.client.force_authenticate(UserFactory.create_user())
self.url = "/api/v2/organization-source-records/"
def transport(self, source):
return (
client_with(fixture("license.json"))
if source == "checko_licenses"
else client_with(
fixture("messages-page-1.json"), fixture("messages-page-2.json")
)
)
def run_task(self, source):
import_module("apps.parsers.tasks")
task = celery_app.tasks[f"parsers.{source}.refresh"]
with patch.object(snapshots, "_client", return_value=self.transport(source)):
return task.apply(throw=True).result
def test_three_independent_groups_and_typed_list_detail_exports(self):
for source, (group, kind) in SOURCES.items():
descriptor = get_source_group_descriptor(source)
assert (descriptor.source_group, descriptor.record_type) == (group, kind)
with patch.object(
snapshots, "_client", return_value=self.transport(source)
):
snapshots.refresh_checko_section(source=source, load_batch=1)
response = self.client.get(self.url, {"source_group": group})
assert response.status_code == 200, response.data
assert response.data["data"], response.data
for row in response.data["data"]:
assert row["source"] == source
assert row["record_type"] == kind
assert row["amount"] is None
assert row["payload"]["fetched_at"].endswith("Z")
assert row["organization"]["inn"] == self.org.inn
detail = self.client.get(self.url + row["uid"] + "/")
assert detail.status_code == 200, detail.data
assert detail.data["payload"] == row["payload"]
record = _source_group_queryset(group, export_year=2026).get(
uid=row["uid"]
)
exported = _build_record_row(record, include_financial_lines=False)
assert exported["source"] == source
assert exported["payload.source_url"] == row["payload"]["source_url"]
if source == "checko_licenses":
assert response.data["data"][0]["payload"]["valid_until"] is None
def test_partial_does_not_move_full_last_success_and_keeps_mixed_batches_visible(
self,
):
source = "checko_messages"
full = self.run_task(source)
assert full["completeness"] == "full"
before = SourceCardService.get_card("checko-messages")["latest_success_load"][
"batch_id"
]
org2 = Organization.objects.create(
name="АО Второй",
inn="7702070139",
ogrn="1027700070518",
okpo="87654321",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
original_collect = snapshots.collect_checko_section
def collected(client, *, organization, source):
# Real adapter, identical structure for the second canonical identity.
responses = [
fixture("messages-page-1.json"),
fixture("messages-page-2.json"),
]
for page in responses:
page["company"] = {"ИНН": org2.inn, "ОГРН": org2.ogrn}
for row in page["data"]["Записи"]:
row["Публикатор"] = org2.ogrn
row["Участники"] = [org2.inn]
yield from original_collect(
client_with(*responses), organization=organization, source=source
)
with patch.object(snapshots, "collect_checko_section", collected):
partial = self.run_task(source)
assert partial["completeness"] == "partial"
assert partial["quota_skipped"] == partial["organizations_preserved"] == 1
assert partial["organizations_updated"] == 1
card = SourceCardService.get_card("checko-messages")
assert card["latest_success_load"]["batch_id"] == before
assert card["latest_load"]["meta"]["completeness"] == "partial"
response = self.client.get(self.url, {"source_group": "company_messages"})
assert response.status_code == 200, response.data
assert {row["load_batch"] for row in response.data["data"]} == {
full["batch_id"],
partial["batch_id"],
}
assert len(response.data["data"]) == 4
assert (
BackgroundJob.objects.order_by("-created_at").first().result["completeness"]
== "partial"
)
def test_no_completed_failure_exposes_rejected_counters(self):
source = "checko_messages"
claim_monthly_collection(organization_id=str(self.org.uid), source=source)
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_task(source)
job = BackgroundJob.objects.get()
assert job.status == "failure"
assert job.result["quota_skipped"] == 1
assert job.result["organizations_preserved"] == 1
assert job.result["completeness"] == "partial"
assert job.result["artifact_id"] == str(ParserSourceArtifact.objects.get().uid)
assert (
SourceCardService.get_card("checko-messages")["latest_success_load"] is None
)
@pytest.mark.django_db
@pytest.mark.parametrize("source", SOURCES)
def test_refresh_and_run_share_guard_permissions_and_no_file_url(source):
import_module("apps.parsers.tasks")
task = celery_app.tasks[f"parsers.{source}.refresh"]
client = APIClient()
client.force_authenticate(UserFactory.create_user())
slug = source.replace("_", "-")
urls = [f"/api/v1/sources/{slug}/refresh/", f"/api/v1/parsers/run/{source}/"]
for url in urls:
assert client.post(url, {}, format="json").status_code == 403
client.force_authenticate(UserFactory.create_user(is_staff=True))
for url in urls:
assert (
client.post(
url, {"params": {"file_url": "https://foreign.invalid"}}, format="json"
).status_code
== 400
)
with patch.object(task, "apply_async"):
assert client.post(urls[0], {}, format="json").status_code == 202
assert client.post(urls[1], {}, format="json").status_code == 409
assert BackgroundJob.objects.count() == 1
def test_checko_discriminated_variants_in_canonical_document():
import json
from pathlib import Path
document = json.loads(
(Path(__file__).parents[3] / "src/core/openapi.json").read_text()
)
schemas = document["components"]["schemas"]
for name in ("OrganizationSourceRecordList", "OrganizationSourceRecord"):
mapping = schemas[name]["discriminator"]["mapping"]
for source, (group, kind) in SOURCES.items():
variant = schemas[mapping[group].rsplit("/", 1)[1]]
assert variant["properties"]["source"]["enum"] == [source]
assert variant["properties"]["record_type"]["enum"] == [kind]

View File

@@ -8,6 +8,7 @@ from uuid import uuid4
import pytest
from apps.parsers.budget_registry import normalize_budget_record
from apps.parsers.checko_sections import collect_checko_section
from apps.parsers.fssp_csv import FSSP_SOURCE, GENERAL_DATASET, iter_fssp_records
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.sme_support import support_payload
@@ -19,6 +20,7 @@ from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_groups import get_source_group_descriptor
from rest_framework.test import APIClient
from tests.apps.parsers.test_checko_sections import client_with, fixture
from tests.apps.parsers.test_registry_payload_contract import full_budget_fixture
from tests.apps.parsers.test_registry_snapshots import _xml
from tests.apps.user.factories import UserFactory
@@ -134,7 +136,34 @@ def _canonical_records():
"payload": row.payload,
},
)
return [budget, sme, sro, fssp], raw_budget
sections = []
for source in ("checko_leasing", "checko_messages", "checko_licenses"):
client = (
client_with(fixture("license.json"))
if source == "checko_licenses"
else client_with(
fixture("messages-page-1.json"), fixture("messages-page-2.json")
)
)
row = next(collect_checko_section(client, organization=fssp_org, source=source))
row.payload["upstream"]["future_nested"] = {
"keep": [None, False, 0, {"unknown": "0007"}]
}
sections.append(
_create_record(
fssp_org,
source,
{
"title": row.title,
"record_date": row.record_date,
"amount": None,
"status": row.status,
"url": row.url,
"payload": row.payload,
},
)
)
return [budget, sme, sro, fssp, *sections], raw_budget
@pytest.mark.django_db
@@ -155,8 +184,8 @@ def test_source_record_runtime_fixtures():
return body
mixed = capture("mixed-list", "list", client.get(list_url))
assert len(mixed["data"]) == 4
assert mixed["meta"]["pagination"]["total_count"] == 4
assert len(mixed["data"]) == 7
assert mixed["meta"]["pagination"]["total_count"] == 7
details = {}
for record in records:
listing = capture(

View File

@@ -69,6 +69,9 @@ def issue(client, group="budget_process_registry", file_format="json"):
@pytest.mark.parametrize(
"source,group",
[
("checko_leasing", "leasing"),
("checko_messages", "company_messages"),
("checko_licenses", "company_licenses"),
("budget_ubpandnubp", "budget_process_registry"),
("fssp_enforcement_proceedings", "enforcement_proceedings"),
],
@@ -217,7 +220,7 @@ def test_sro_export_keeps_all_years_and_identifier_strings(export_client, file_f
record_date="2001-01-01",
)
generation = exports.build_source_record_export_artifacts()
assert generation.artifacts_count == 52
assert generation.artifacts_count == 61
response = issue(export_client, "sro_membership", file_format)
assert response.status_code == 201
package = exports.consume_source_record_export_download_ticket(

View File

@@ -128,9 +128,9 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
call_command("build_source_record_exports", stdout=command_output)
generation = load_current_source_record_export_generation()
self.assertEqual(generation.artifacts_count, 52)
self.assertEqual(generation.artifacts_count, 61)
self.assertEqual(generation.export_year, timezone.localdate().year)
self.assertIn('"artifacts_count": 52', command_output.getvalue())
self.assertIn('"artifacts_count": 61', command_output.getvalue())
def test_sanctions_xlsx_export_uses_source_contract_columns_and_boolean_flags(self):
organization = Organization.objects.create(
@@ -641,7 +641,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertEqual(
response["X-Source-Export-Generated-At"], generation.generated_at
)
self.assertEqual(generation.artifacts_count, 52)
self.assertEqual(generation.artifacts_count, 61)
self.assertIn(
'filename="planned-inspections__financial-indicators_',
response["Content-Disposition"],
@@ -840,8 +840,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
key=lambda item: item.file_name,
)
self.assertEqual(generation.artifacts_count, 52)
self.assertEqual(generation.files_count, 53)
self.assertEqual(generation.artifacts_count, 61)
self.assertEqual(generation.files_count, 62)
self.assertEqual(
[item.file_name for item in artifacts],
[
@@ -1065,7 +1065,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
first_generation = build_source_record_export_artifacts()
current_generation = load_current_source_record_export_generation()
self.assertEqual(first_generation.artifacts_count, 52)
self.assertEqual(first_generation.artifacts_count, 61)
self.assertEqual(
current_generation.generation_id, first_generation.generation_id
)

View File

@@ -147,7 +147,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase):
result = refresh_source_record_export_artifacts()
self.assertEqual(result["status"], "success")
self.assertEqual(result["artifacts_count"], 52)
self.assertEqual(result["artifacts_count"], 61)
self.assertEqual(result["export_year"], timezone.localdate().year)
self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))

View File

@@ -80,6 +80,9 @@ class TestCompaniesCommandsTest(TestCase):
ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS,
ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,
ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS,
ParserLoadLog.Source.CHECKO_LEASING,
ParserLoadLog.Source.CHECKO_MESSAGES,
ParserLoadLog.Source.CHECKO_LICENSES,
}
}
for company in companies:
@@ -274,7 +277,7 @@ class TestCompaniesCommandsTest(TestCase):
)
self.assertEqual(
OrganizationSourceRecord.objects.count(),
20 * (len(ParserLoadLog.Source) - 4),
20 * (len(ParserLoadLog.Source) - 7),
)
def test_create_replaces_stale_source_record_for_same_legacy_row(self):

View File

@@ -31,6 +31,9 @@ def _parallel(actions):
@pytest.mark.parametrize(
"slug,source",
[
("checko-leasing", "checko_leasing"),
("checko-messages", "checko_messages"),
("checko-licenses", "checko_licenses"),
("sme-support-recipients-registry", "fns_sme_support_recipients"),
("budget-process-registry", "budget_ubpandnubp"),
("sro-membership-check", "sro_membership_check"),

View File

@@ -97,6 +97,9 @@ class SourceCardServiceUnitTest(TestCase):
"sro-membership-check",
"integrated-structure-documents",
"fedresurs-company-profile",
"checko-leasing",
"checko-messages",
"checko-licenses",
"fssp-enforcement-proceedings",
],
)
@@ -120,6 +123,9 @@ class SourceCardServiceUnitTest(TestCase):
"Проверка членства в СРО",
"Документы интегрированных структур",
"Федресурс: сведения о юридических лицах",
"Лизинг",
"Сообщения организаций",
"Лицензии организаций",
"ФССП: Банк данных исполнительных производств",
],
)

View File

@@ -50,7 +50,8 @@ class ParserSourceRegistryPresentationTest(SimpleTestCase):
public_metadata = " ".join(
str(value)
for descriptor in PARSER_SOURCES.values()
for value in asdict(descriptor).values()
for field, value in asdict(descriptor).items()
if field not in {"key", "source", "task_name"}
)
self.assertNotIn("checko", public_metadata.lower())