fix: preserve complete source snapshots and exports
This commit is contained in:
@@ -234,7 +234,10 @@ def _messages(client, organization, source, fetched_at):
|
||||
if received > pagination.total_records:
|
||||
raise SnapshotValidationError("inconsistent_checko_record_count")
|
||||
if page == pagination.total_pages:
|
||||
if received != pagination.total_records:
|
||||
if (
|
||||
received != pagination.total_records
|
||||
or len(seen) != pagination.total_records
|
||||
):
|
||||
raise SnapshotValidationError("incomplete_checko_record_count")
|
||||
return
|
||||
raise SnapshotValidationError("checko_pagination_limit_exceeded")
|
||||
|
||||
@@ -195,6 +195,8 @@ class _Collector:
|
||||
raise SnapshotValidationError("duplicate_or_invalid_moex_security")
|
||||
seen.add(secid)
|
||||
yield row
|
||||
if total is not None and start + len(rows) == total:
|
||||
return
|
||||
raise SnapshotValidationError("moex_page_limit")
|
||||
|
||||
|
||||
|
||||
@@ -49,6 +49,7 @@ EXPORT_FORMAT_JSON = "json"
|
||||
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
|
||||
FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value
|
||||
ALL_HISTORY_SOURCE_GROUPS = {
|
||||
SourceGroup.ENFORCEMENT_PROCEEDINGS.value,
|
||||
"stock_exchange_activity",
|
||||
"leasing",
|
||||
"company_messages",
|
||||
|
||||
@@ -105,6 +105,24 @@ class FsspContractTests(TestCase):
|
||||
assert response.data["data"][0]["status"] == "ended"
|
||||
assert response.data["data"][1]["payload"]["terminated_on"] is None
|
||||
|
||||
def test_current_export_keeps_previous_and_future_initiation_years(self):
|
||||
from organizations.models import OrganizationSourceRecord
|
||||
|
||||
rows = list(OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE))
|
||||
for row, initiated in zip(
|
||||
rows, [date(2025, 1, 1), date(2027, 1, 1)], strict=True
|
||||
):
|
||||
row.record_date = initiated
|
||||
row.payload["initiated_on"] = initiated.isoformat()
|
||||
row.save(update_fields=["record_date", "payload"])
|
||||
response = self.client.get(self.url, {"source": FSSP_SOURCE})
|
||||
assert response.status_code == 200
|
||||
assert set(
|
||||
_source_group_queryset(
|
||||
"enforcement_proceedings", export_year=2026
|
||||
).values_list("uid", flat=True)
|
||||
) == {UUID(row["uid"]) for row in response.data["data"]}
|
||||
|
||||
def test_snapshot_history_uses_aggregate_not_latest_input(self):
|
||||
log = ParserLoadLog.objects.create(source=FSSP_SOURCE, batch_id=1)
|
||||
assert get_snapshot_artifact(log).uid == self.artifact.uid
|
||||
|
||||
@@ -181,6 +181,49 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
assert artifact.metadata["quota_skipped"] == 2
|
||||
assert before == [self.existing(org) for org in self.orgs]
|
||||
|
||||
def test_overlapping_provider_pages_preserve_previous_four_records(self):
|
||||
from copy import deepcopy
|
||||
|
||||
from apps.parsers.checko_sections import collect_checko_section
|
||||
|
||||
from tests.apps.parsers.test_checko_sections import client_with, fixture
|
||||
|
||||
org = self.orgs[0]
|
||||
OrganizationSourceIngestionService.save_records(
|
||||
source=SOURCE,
|
||||
load_batch=1,
|
||||
records=[record(org, str(n)) for n in range(4)],
|
||||
)
|
||||
before = self.existing(org)
|
||||
first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json")
|
||||
shared = deepcopy(first["data"]["Записи"][0])
|
||||
first["data"]["Записи"].append(dict(shared, GUID="other-first"))
|
||||
second["data"]["Записи"].insert(0, shared)
|
||||
for response in (first, second):
|
||||
response["company"].update({"ИНН": org.inn, "ОГРН": org.ogrn})
|
||||
response["data"]["ЗапВсего"] = 4
|
||||
for message in response["data"]["Записи"]:
|
||||
message["Публикатор"] = org.ogrn
|
||||
message["Участники"] = [org.inn]
|
||||
rows = collect_checko_section(
|
||||
client_with(first, second),
|
||||
organization=org,
|
||||
source=SOURCE,
|
||||
)
|
||||
(artifact, _), _ = self.run_refresh(rows=[rows, [record(self.orgs[1])]])
|
||||
assert self.existing(org) == before
|
||||
assert artifact.metadata["organizations_failed"] == 1
|
||||
assert artifact.metadata["completeness"] == "partial"
|
||||
assert not ParserStagedRecord.objects.filter(
|
||||
artifact=artifact, organization=org
|
||||
).exists()
|
||||
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
|
||||
assert attempt.status == attempt.Status.FAILED
|
||||
assert (
|
||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_publication_failure_rolls_back_records(self):
|
||||
self.seed(self.orgs[0])
|
||||
before = self.existing(self.orgs[0])
|
||||
|
||||
@@ -165,6 +165,21 @@ def test_repeated_page_content_with_advanced_number_rejected():
|
||||
collect(client_with(first, second))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["checko_messages", "checko_leasing"])
|
||||
def test_overlapping_pages_cannot_prove_total_before_leasing_filter(source):
|
||||
from copy import deepcopy
|
||||
|
||||
first, second = fixture("messages-page-1.json"), fixture("messages-page-2.json")
|
||||
shared = deepcopy(first["data"]["Записи"][0])
|
||||
first["data"]["Записи"].append(dict(shared, GUID="other-first"))
|
||||
second["data"]["Записи"].insert(0, shared)
|
||||
for response in (first, second):
|
||||
response["data"]["ЗапВсего"] = 4
|
||||
with pytest.raises(SnapshotValidationError) as error:
|
||||
collect(client_with(first, second), source)
|
||||
assert str(error.value.__cause__) == "incomplete_checko_record_count"
|
||||
|
||||
|
||||
def test_license_unknown_raw_fields_survive():
|
||||
response = fixture("license.json")
|
||||
response["data"]["Лиценз"][0]["FutureField"] = {"nested": [None, 0, False]}
|
||||
|
||||
@@ -235,6 +235,33 @@ def test_unknown_columns_rejected():
|
||||
adapter().iss_rows({"x": block(["wrong"], [])}, "x", {"secid"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("total", [1, 100, 101])
|
||||
def test_cursor_stops_after_complete_final_page(total):
|
||||
from io import BytesIO
|
||||
from unittest.mock import Mock
|
||||
|
||||
module = adapter()
|
||||
collector = module._Collector(None, BytesIO())
|
||||
columns = sorted(module.REFERENCE_COLUMNS)
|
||||
|
||||
def read(url, params):
|
||||
start = params["start"]
|
||||
rows = [dict.fromkeys(columns) for _ in range(start, min(start + 100, total))]
|
||||
for n, row in enumerate(rows, start):
|
||||
row["secid"] = f"S{n}"
|
||||
return {
|
||||
"securities": block(columns, [[r[c] for c in columns] for r in rows]),
|
||||
"securities.cursor": block(
|
||||
["INDEX", "TOTAL", "PAGESIZE"], [[start, total, 100]]
|
||||
),
|
||||
}
|
||||
|
||||
collector.read = Mock(side_effect=read)
|
||||
rows = list(collector.reference("bonds"))
|
||||
assert len(rows) == total
|
||||
assert collector.read.call_count == (total + 99) // 100
|
||||
|
||||
|
||||
def test_outside_directory_is_counted_without_parameter_requests(scenario):
|
||||
scenario[0].delete()
|
||||
scenario[3].by_inn.pop("7707083893")
|
||||
|
||||
Reference in New Issue
Block a user