fix: preserve complete source snapshots and exports

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:49:22 +02:00
parent 3ce9898c0e
commit b1710408fd
7 changed files with 110 additions and 1 deletions

View File

@@ -234,7 +234,10 @@ def _messages(client, organization, source, fetched_at):
if received > pagination.total_records:
raise SnapshotValidationError("inconsistent_checko_record_count")
if page == pagination.total_pages:
if received != pagination.total_records:
if (
received != pagination.total_records
or len(seen) != pagination.total_records
):
raise SnapshotValidationError("incomplete_checko_record_count")
return
raise SnapshotValidationError("checko_pagination_limit_exceeded")

View File

@@ -195,6 +195,8 @@ class _Collector:
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
seen.add(secid)
yield row
if total is not None and start + len(rows) == total:
return
raise SnapshotValidationError("moex_page_limit")

View File

@@ -49,6 +49,7 @@ EXPORT_FORMAT_JSON = "json"
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
ALL_HISTORY_SOURCE_GROUPS = {
SourceGroup.ENFORCEMENT_PROCEEDINGS.value,
"stock_exchange_activity",
"leasing",
"company_messages",

View File

@@ -105,6 +105,24 @@ class FsspContractTests(TestCase):
assert response.data["data"][0]["status"] == "ended"
assert response.data["data"][1]["payload"]["terminated_on"] is None
def test_current_export_keeps_previous_and_future_initiation_years(self):
from organizations.models import OrganizationSourceRecord
rows = list(OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE))
for row, initiated in zip(
rows, [date(2025, 1, 1), date(2027, 1, 1)], strict=True
):
row.record_date = initiated
row.payload["initiated_on"] = initiated.isoformat()
row.save(update_fields=["record_date", "payload"])
response = self.client.get(self.url, {"source": FSSP_SOURCE})
assert response.status_code == 200
assert set(
_source_group_queryset(
"enforcement_proceedings", export_year=2026
).values_list("uid", flat=True)
) == {UUID(row["uid"]) for row in response.data["data"]}
def test_snapshot_history_uses_aggregate_not_latest_input(self):
log = ParserLoadLog.objects.create(source=FSSP_SOURCE, batch_id=1)
assert get_snapshot_artifact(log).uid == self.artifact.uid

View File

@@ -181,6 +181,49 @@ class CheckoSectionSnapshotTests(TestCase):
assert artifact.metadata["quota_skipped"] == 2
assert before == [self.existing(org) for org in self.orgs]
def test_overlapping_provider_pages_preserve_previous_four_records(self):
from copy import deepcopy
from apps.parsers.checko_sections import collect_checko_section
from tests.apps.parsers.test_checko_sections import client_with, fixture
org = self.orgs[0]
OrganizationSourceIngestionService.save_records(
source=SOURCE,
load_batch=1,
records=[record(org, str(n)) for n in range(4)],
)
before = self.existing(org)
first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json")
shared = deepcopy(first["data"]["Записи"][0])
first["data"]["Записи"].append(dict(shared, GUID="other-first"))
second["data"]["Записи"].insert(0, shared)
for response in (first, second):
response["company"].update({"ИНН": org.inn, "ОГРН": org.ogrn})
response["data"]["ЗапВсего"] = 4
for message in response["data"]["Записи"]:
message["Публикатор"] = org.ogrn
message["Участники"] = [org.inn]
rows = collect_checko_section(
client_with(first, second),
organization=org,
source=SOURCE,
)
(artifact, _), _ = self.run_refresh(rows=[rows, [record(self.orgs[1])]])
assert self.existing(org) == before
assert artifact.metadata["organizations_failed"] == 1
assert artifact.metadata["completeness"] == "partial"
assert not ParserStagedRecord.objects.filter(
artifact=artifact, organization=org
).exists()
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
is None
)
def test_publication_failure_rolls_back_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])

View File

@@ -165,6 +165,21 @@ def test_repeated_page_content_with_advanced_number_rejected():
collect(client_with(first, second))
@pytest.mark.parametrize("source", ["checko_messages", "checko_leasing"])
def test_overlapping_pages_cannot_prove_total_before_leasing_filter(source):
from copy import deepcopy
first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json")
shared = deepcopy(first["data"]["Записи"][0])
first["data"]["Записи"].append(dict(shared, GUID="other-first"))
second["data"]["Записи"].insert(0, shared)
for response in (first, second):
response["data"]["ЗапВсего"] = 4
with pytest.raises(SnapshotValidationError) as error:
collect(client_with(first, second), source)
assert str(error.value.__cause__) == "incomplete_checko_record_count"
def test_license_unknown_raw_fields_survive():
response = fixture("license.json")
response["data"]["Лиценз"][0]["FutureField"] = {"nested": [None, 0, False]}

View File

@@ -235,6 +235,33 @@ def test_unknown_columns_rejected():
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
@pytest.mark.parametrize("total", [1, 100, 101])
def test_cursor_stops_after_complete_final_page(total):
from io import BytesIO
from unittest.mock import Mock
module = adapter()
collector = module._Collector(None, BytesIO())
columns = sorted(module.REFERENCE_COLUMNS)
def read(url, params):
start = params["start"]
rows = [dict.fromkeys(columns) for _ in range(start, min(start + 100, total))]
for n, row in enumerate(rows, start):
row["secid"] = f"S{n}"
return {
"securities": block(columns, [[r[c] for c in columns] for r in rows]),
"securities.cursor": block(
["INDEX", "TOTAL", "PAGESIZE"], [[start, total, 100]]
),
}
collector.read = Mock(side_effect=read)
rows = list(collector.reference("bonds"))
assert len(rows) == total
assert collector.read.call_count == (total + 99) // 100
def test_outside_directory_is_counted_without_parameter_requests(scenario):
scenario[0].delete()
scenario[3].by_inn.pop("7707083893")