From b1710408fd4e094e83e2ace65d8bf73d4d411c21 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 5 Oct 2026 17:49:22 +0200 Subject: [PATCH] fix: preserve complete source snapshots and exports --- src/apps/parsers/checko_sections.py | 5 ++- src/apps/parsers/exchange_moex.py | 2 + src/organizations/source_record_export.py | 1 + .../apps/organizations/test_fssp_contract.py | 18 ++++++++ .../parsers/test_checko_section_snapshots.py | 43 +++++++++++++++++++ tests/apps/parsers/test_checko_sections.py | 15 +++++++ tests/apps/parsers/test_exchange_moex.py | 27 ++++++++++++ 7 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/apps/parsers/checko_sections.py b/src/apps/parsers/checko_sections.py index f28b13e..3422577 100644 --- a/src/apps/parsers/checko_sections.py +++ b/src/apps/parsers/checko_sections.py @@ -234,7 +234,10 @@ def _messages(client, organization, source, fetched_at): if received > pagination.total_records: raise SnapshotValidationError("inconsistent_checko_record_count") if page == pagination.total_pages: - if received != pagination.total_records: + if ( + received != pagination.total_records + or len(seen) != pagination.total_records + ): raise SnapshotValidationError("incomplete_checko_record_count") return raise SnapshotValidationError("checko_pagination_limit_exceeded") diff --git a/src/apps/parsers/exchange_moex.py b/src/apps/parsers/exchange_moex.py index f200430..5f54c31 100644 --- a/src/apps/parsers/exchange_moex.py +++ b/src/apps/parsers/exchange_moex.py @@ -195,6 +195,8 @@ class _Collector: raise SnapshotValidationError("duplicate_or_invalid_moex_security") seen.add(secid) yield row + if total is not None and start + len(rows) == total: + return raise SnapshotValidationError("moex_page_limit") diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index dc0c368..e607ef9 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -49,6 +49,7 @@ EXPORT_FORMAT_JSON = "json" EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON) FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value ALL_HISTORY_SOURCE_GROUPS = { + SourceGroup.ENFORCEMENT_PROCEEDINGS.value, "stock_exchange_activity", "leasing", "company_messages", diff --git a/tests/apps/organizations/test_fssp_contract.py b/tests/apps/organizations/test_fssp_contract.py index df88f20..d12968c 100644 --- a/tests/apps/organizations/test_fssp_contract.py +++ b/tests/apps/organizations/test_fssp_contract.py @@ -105,6 +105,24 @@ class FsspContractTests(TestCase): assert response.data["data"][0]["status"] == "ended" assert response.data["data"][1]["payload"]["terminated_on"] is None + def test_current_export_keeps_previous_and_future_initiation_years(self): + from organizations.models import OrganizationSourceRecord + + rows = list(OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE)) + for row, initiated in zip( + rows, [date(2025, 1, 1), date(2027, 1, 1)], strict=True + ): + row.record_date = initiated + row.payload["initiated_on"] = initiated.isoformat() + row.save(update_fields=["record_date", "payload"]) + response = self.client.get(self.url, {"source": FSSP_SOURCE}) + assert response.status_code == 200 + assert set( + _source_group_queryset( + "enforcement_proceedings", export_year=2026 + ).values_list("uid", flat=True) + ) == {UUID(row["uid"]) for row in response.data["data"]} + def test_snapshot_history_uses_aggregate_not_latest_input(self): log = ParserLoadLog.objects.create(source=FSSP_SOURCE, batch_id=1) assert get_snapshot_artifact(log).uid == self.artifact.uid diff --git a/tests/apps/parsers/test_checko_section_snapshots.py b/tests/apps/parsers/test_checko_section_snapshots.py index d2ca297..d72b9c8 100644 --- a/tests/apps/parsers/test_checko_section_snapshots.py +++ b/tests/apps/parsers/test_checko_section_snapshots.py @@ -181,6 +181,49 @@ class CheckoSectionSnapshotTests(TestCase): assert artifact.metadata["quota_skipped"] == 2 assert before == [self.existing(org) for org in self.orgs] + def test_overlapping_provider_pages_preserve_previous_four_records(self): + from copy import deepcopy + + from apps.parsers.checko_sections import collect_checko_section + + from tests.apps.parsers.test_checko_sections import client_with, fixture + + org = self.orgs[0] + OrganizationSourceIngestionService.save_records( + source=SOURCE, + load_batch=1, + records=[record(org, str(n)) for n in range(4)], + ) + before = self.existing(org) + first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json") + shared = deepcopy(first["data"]["Записи"][0]) + first["data"]["Записи"].append(dict(shared, GUID="other-first")) + second["data"]["Записи"].insert(0, shared) + for response in (first, second): + response["company"].update({"ИНН": org.inn, "ОГРН": org.ogrn}) + response["data"]["ЗапВсего"] = 4 + for message in response["data"]["Записи"]: + message["Публикатор"] = org.ogrn + message["Участники"] = [org.inn] + rows = collect_checko_section( + client_with(first, second), + organization=org, + source=SOURCE, + ) + (artifact, _), _ = self.run_refresh(rows=[rows, [record(self.orgs[1])]]) + assert self.existing(org) == before + assert artifact.metadata["organizations_failed"] == 1 + assert artifact.metadata["completeness"] == "partial" + assert not ParserStagedRecord.objects.filter( + artifact=artifact, organization=org + ).exists() + attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE) + assert attempt.status == attempt.Status.FAILED + assert ( + claim_monthly_collection(organization_id=str(org.uid), source=SOURCE) + is None + ) + def test_publication_failure_rolls_back_records(self): self.seed(self.orgs[0]) before = self.existing(self.orgs[0]) diff --git a/tests/apps/parsers/test_checko_sections.py b/tests/apps/parsers/test_checko_sections.py index 6a1e16c..a97497c 100644 --- a/tests/apps/parsers/test_checko_sections.py +++ b/tests/apps/parsers/test_checko_sections.py @@ -165,6 +165,21 @@ def test_repeated_page_content_with_advanced_number_rejected(): collect(client_with(first, second)) +@pytest.mark.parametrize("source", ["checko_messages", "checko_leasing"]) +def test_overlapping_pages_cannot_prove_total_before_leasing_filter(source): + from copy import deepcopy + + first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json") + shared = deepcopy(first["data"]["Записи"][0]) + first["data"]["Записи"].append(dict(shared, GUID="other-first")) + second["data"]["Записи"].insert(0, shared) + for response in (first, second): + response["data"]["ЗапВсего"] = 4 + with pytest.raises(SnapshotValidationError) as error: + collect(client_with(first, second), source) + assert str(error.value.__cause__) == "incomplete_checko_record_count" + + def test_license_unknown_raw_fields_survive(): response = fixture("license.json") response["data"]["Лиценз"][0]["FutureField"] = {"nested": [None, 0, False]} diff --git a/tests/apps/parsers/test_exchange_moex.py b/tests/apps/parsers/test_exchange_moex.py index c71791e..1f3316a 100644 --- a/tests/apps/parsers/test_exchange_moex.py +++ b/tests/apps/parsers/test_exchange_moex.py @@ -235,6 +235,33 @@ def test_unknown_columns_rejected(): adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"}) +@pytest.mark.parametrize("total", [1, 100, 101]) +def test_cursor_stops_after_complete_final_page(total): + from io import BytesIO + from unittest.mock import Mock + + module = adapter() + collector = module._Collector(None, BytesIO()) + columns = sorted(module.REFERENCE_COLUMNS) + + def read(url, params): + start = params["start"] + rows = [dict.fromkeys(columns) for _ in range(start, min(start + 100, total))] + for n, row in enumerate(rows, start): + row["secid"] = f"S{n}" + return { + "securities": block(columns, [[r[c] for c in columns] for r in rows]), + "securities.cursor": block( + ["INDEX", "TOTAL", "PAGESIZE"], [[start, total, 100]] + ), + } + + collector.read = Mock(side_effect=read) + rows = list(collector.reference("bonds")) + assert len(rows) == total + assert collector.read.call_count == (total + 99) // 100 + + def test_outside_directory_is_counted_without_parameter_requests(scenario): scenario[0].delete() scenario[3].by_inn.pop("7707083893")