Files
mostovik-backend/tests/apps/parsers/test_fssp_snapshot.py
2026-10-05 16:00:54 +02:00

310 lines
12 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Atomic FSSP publication: real staging/database, synthetic input files."""
import csv
import importlib
from datetime import date
from io import BytesIO, StringIO
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import patch
import pytest
import requests
from apps.parsers.fssp_csv import ENDED_DATASET, FSSP_SOURCE, GENERAL_DATASET
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from apps.parsers.registry_snapshots import SnapshotValidationError
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
FIXTURES = Path(__file__).parents[2] / "fixtures" / "fssp"
def adapter():
try:
return importlib.import_module("apps.parsers.fssp_snapshot")
except ModuleNotFoundError:
pytest.fail("FSSP snapshot importer is not implemented")
def csv_bytes(name, **changes):
rows = list(csv.reader(StringIO((FIXTURES / name).read_text())))
for key, value in changes.items():
rows[1][rows[0].index(key)] = value
handle = StringIO()
csv.writer(handle).writerows(rows)
return handle.getvalue().encode()
class FsspSnapshotTests(TestCase):
def setUp(self):
self.tmp = TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.settings = override_settings(
MEDIA_ROOT=self.tmp.name + "/public",
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
)
self.settings.enable()
self.addCleanup(self.settings.disable)
self.org = Organization.objects.create(
name="АО Проверка",
inn="7707083893",
ogrn="1027700132195",
okpo="12345678",
directory_imported_at=timezone.now(),
)
def run_import(self, general=None, ended=None, batch=1, **kwargs):
module = adapter()
inputs = [
module.FsspInput(
GENERAL_DATASET,
BytesIO(general or csv_bytes("general.csv")),
"general.csv",
date(2026, 10, 5),
),
module.FsspInput(
ENDED_DATASET,
BytesIO(ended or csv_bytes("ended.csv")),
"ended.csv",
date(2026, 10, 5),
),
]
return module.import_fssp_snapshots(inputs=inputs, load_batch=batch, **kwargs)
def published(self):
return list(
OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE).values_list(
"uid", "payload"
)
)
def test_invalid_canonical_requisites_are_quarantined(self):
self.org.ogrn = "bad"
self.org.okpo = "1234567890"
self.org.save()
artifact, result = self.run_import()
assert result.published == 0
assert result.quarantined == 2
assert artifact.metadata["reasons"]["invalid_canonical_requisites"] == 2
def test_two_inputs_manifest_and_stable_uid(self):
artifact, result = self.run_import()
assert result.parsed == 2
assert result.published == 1
assert len(artifact.metadata["inputs"]) == 2
row = OrganizationSourceRecord.objects.get(source=FSSP_SOURCE)
assert row.status == "ended"
assert row.amount is None
assert len(row.payload["observations"]) == 2
assert row.extension.organization_id == self.org.uid
uid = row.uid
self.run_import(batch=2)
assert OrganizationSourceRecord.objects.get(source=FSSP_SOURCE).uid == uid
assert Organization.objects.count() == 1
assert row.legacy_model == ""
assert row.legacy_pk == ""
with pytest.raises(ValueError):
_ = artifact.file.url
def test_second_input_failure_preserves_snapshot(self):
self.run_import()
before = self.published()
with pytest.raises(SnapshotValidationError):
self.run_import(ended=b"invalid,csv\n", batch=2)
assert self.published() == before
assert (
ParserSourceArtifact.objects.filter(
status=ParserSourceArtifact.Status.PUBLISHED
).count()
== 1
)
def test_conflicting_duplicate_blocks_publication(self):
self.run_import()
before = self.published()
good = csv_bytes("general.csv")
changed = csv_bytes("general.csv", **{"Debt remaining balance": "100"})
rows = list(csv.reader(StringIO(changed.decode())))
handle = StringIO()
csv.writer(handle).writerow(rows[1])
with pytest.raises(SnapshotValidationError, match="conflicting_fssp_identity"):
self.run_import(general=good + handle.getvalue().encode(), batch=2)
assert self.published() == before
def test_foreign_inn_skipped_and_invalid_inn_quarantined(self):
general = csv_bytes("general.csv", **{"Debtor TIN": "7704228075"})
ended = csv_bytes("ended.csv", **{"Debtor TIN": "7707083894"})
artifact, result = self.run_import(general=general, ended=ended)
assert result.skipped == 1
assert result.quarantined == 1
assert result.published == 0
assert Organization.objects.count() == 1
assert artifact.metadata["reasons"]["invalid_debtor_inn"] == 1
assert artifact.staged_records.count() == 1
def test_ambiguous_identity_quarantined(self):
Organization.objects.create(
name="Другой филиал",
inn=self.org.inn,
kpp="111111111",
ogrn="1027700132195",
okpo="12345679",
directory_imported_at=timezone.now(),
)
artifact, result = self.run_import()
assert result.published == 0
assert result.quarantined == 2
assert artifact.metadata["reasons"]["ambiguous_organization"] == 2
def test_missing_one_input_rejected(self):
module = adapter()
with pytest.raises(SnapshotValidationError, match="missing_fssp_input"):
module.import_fssp_snapshots(inputs=[], load_batch=1)
assert not OrganizationSourceRecord.objects.exists()
def test_publication_callback_failure_rolls_back_records(self):
self.run_import()
before = self.published()
def fail(artifact, result):
raise RuntimeError("fixture publication failure")
with pytest.raises(RuntimeError):
self.run_import(batch=2, on_publish=fail)
assert self.published() == before
assert (
ParserStagedRecord.objects.filter(
disposition="published", artifact__load_batch=2
).count()
== 0
)
def test_valid_empty_inputs_clear_previous_snapshot(self):
self.run_import()
general = csv_bytes("general.csv").split(b"\r\n", 1)[0] + b"\r\n"
ended = csv_bytes("ended.csv").split(b"\r\n", 1)[0] + b"\r\n"
artifact, result = self.run_import(general=general, ended=ended, batch=2)
assert result.published == 0
assert not self.published()
assert artifact.status == "published"
def test_progress_includes_rows_outside_directory(self):
rows = list(
csv.reader(
StringIO(
csv_bytes("general.csv", **{"Debtor TIN": "7704228075"}).decode()
)
)
)
handle = StringIO()
csv.writer(handle).writerows([rows[0]] + [rows[1]] * 10000)
progress = []
artifact, result = self.run_import(
general=handle.getvalue().encode(),
on_progress=lambda *counts: progress.append(counts),
)
assert progress and progress[-1][0] == 10000
assert result.skipped == 10000
def test_more_than_one_staging_batch_publishes_all(self):
rows = list(csv.reader(StringIO(csv_bytes("general.csv").decode())))
handle = StringIO()
writer = csv.writer(handle)
writer.writerow(rows[0])
number_column = rows[0].index("The number of enforcement proceeding")
for number in range(501):
row = list(rows[1])
row[number_column] = f"{number}/26/77001-ИП"
writer.writerow(row)
artifact, result = self.run_import(general=handle.getvalue().encode())
assert result.parsed == 502
assert result.published == 501
assert artifact.metadata["deduplicated_records_count"] == 1
assert artifact.staged_records.filter(disposition="published").count() == 501
def test_refresh_downloaded_artifacts_link_to_aggregate(self):
module = adapter()
with patch.object(module, "registry_session", return_value=DownloadSession()):
artifact, result = module.refresh_fssp_enforcement_proceedings(load_batch=1)
assert result.published == 1
for item in artifact.metadata["inputs"]:
raw = ParserSourceArtifact.objects.get(pk=item["artifact_id"])
assert raw.metadata["aggregate_artifact_id"] == str(artifact.uid)
assert raw.sha256 == item["checksum_sha256"]
def test_failed_second_download_retains_first_raw_and_old_snapshot(self):
self.run_import()
before = self.published()
module = adapter()
with patch.object(
module, "registry_session", return_value=DownloadSession(fail_ended=True)
), pytest.raises(requests.HTTPError):
module.refresh_fssp_enforcement_proceedings(load_batch=2)
assert self.published() == before
raw = ParserSourceArtifact.objects.get(load_batch=2)
assert raw.status == "rejected"
assert raw.file.size > 0
class PassportSession:
def __init__(self, body):
self.body = body
def get(self, url, **kwargs):
response = requests.Response()
response.status_code = 200
response._content = self.body
response._content_consumed = True
response.headers["Content-Type"] = "text/csv"
return response
class DownloadSession:
def __init__(self, fail_ended=False):
self.fail_ended = fail_ended
def __enter__(self):
return self
def __exit__(self, *args):
return None
def get(self, url, **kwargs):
ended = ENDED_DATASET in url
if url.endswith("/meta.csv"):
name = "ended-meta.csv" if ended else "general-meta.csv"
return PassportSession((FIXTURES / name).read_bytes()).get(url)
body = csv_bytes("ended.csv" if ended else "general.csv")
response = PassportSession(body).get(url)
response.status_code = 503 if ended and self.fail_ended else 206
response.headers["Content-Range"] = f"bytes 0-{len(body) - 1}/{len(body)}"
response.headers["ETag"] = '"fixture"'
response.url = url
return response
def test_passport_discovery_selects_latest_timestamp_not_row_order():
body = (FIXTURES / "general-meta.csv").read_text()
lines = body.splitlines()
session = PassportSession(("\n".join(reversed(lines[1:])) + "\n").encode())
with pytest.raises(SnapshotValidationError):
adapter().discover_fssp_input(session, GENERAL_DATASET)
session = PassportSession(
("property,value\n" + "\n".join(reversed(lines[1:]))).encode()
)
url, published_at, version = adapter().discover_fssp_input(session, GENERAL_DATASET)
assert url.endswith("data-20261005T0655-structure-20260820T2100.csv")
assert published_at == date(2026, 10, 5)
assert version == "20260820T2100"
def test_passport_external_download_host_rejected():
body = (
(FIXTURES / "general-meta.csv")
.read_text()
.replace("https://opendata.fssp.gov.ru/", "https://foreign.example/")
)
with pytest.raises(SnapshotValidationError):
adapter().discover_fssp_input(PassportSession(body.encode()), GENERAL_DATASET)