fix: preserve complete source snapshots and exports

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:49:22 +02:00
parent 3ce9898c0e
commit b1710408fd
7 changed files with 110 additions and 1 deletions

View File

@@ -234,7 +234,10 @@ def _messages(client, organization, source, fetched_at):
if received > pagination.total_records:
raise SnapshotValidationError("inconsistent_checko_record_count")
if page == pagination.total_pages:
if received != pagination.total_records:
if (
received != pagination.total_records
or len(seen) != pagination.total_records
):
raise SnapshotValidationError("incomplete_checko_record_count")
return
raise SnapshotValidationError("checko_pagination_limit_exceeded")

View File

@@ -195,6 +195,8 @@ class _Collector:
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
seen.add(secid)
yield row
if total is not None and start + len(rows) == total:
return
raise SnapshotValidationError("moex_page_limit")

View File

@@ -49,6 +49,7 @@ EXPORT_FORMAT_JSON = "json"
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
ALL_HISTORY_SOURCE_GROUPS = {
SourceGroup.ENFORCEMENT_PROCEEDINGS.value,
"stock_exchange_activity",
"leasing",
"company_messages",