"""Atomic FSSP publication: real staging/database, synthetic input files.""" import csv import importlib from datetime import date from io import BytesIO, StringIO from pathlib import Path from tempfile import TemporaryDirectory from unittest.mock import patch import pytest import requests from apps.parsers.fssp_csv import ENDED_DATASET, FSSP_SOURCE, GENERAL_DATASET from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord from apps.parsers.registry_snapshots import SnapshotValidationError from django.test import TestCase, override_settings from django.utils import timezone from organizations.models import Organization, OrganizationSourceRecord FIXTURES = Path(__file__).parents[2] / "fixtures" / "fssp" def adapter(): try: return importlib.import_module("apps.parsers.fssp_snapshot") except ModuleNotFoundError: pytest.fail("FSSP snapshot importer is not implemented") def csv_bytes(name, **changes): rows = list(csv.reader(StringIO((FIXTURES / name).read_text()))) for key, value in changes.items(): rows[1][rows[0].index(key)] = value handle = StringIO() csv.writer(handle).writerows(rows) return handle.getvalue().encode() class FsspSnapshotTests(TestCase): def setUp(self): self.tmp = TemporaryDirectory() self.addCleanup(self.tmp.cleanup) self.settings = override_settings( MEDIA_ROOT=self.tmp.name + "/public", PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private", ) self.settings.enable() self.addCleanup(self.settings.disable) self.org = Organization.objects.create( name="АО Проверка", inn="7707083893", ogrn="1027700132195", okpo="12345678", directory_imported_at=timezone.now(), ) def run_import(self, general=None, ended=None, batch=1, **kwargs): module = adapter() inputs = [ module.FsspInput( GENERAL_DATASET, BytesIO(general or csv_bytes("general.csv")), "general.csv", date(2026, 10, 5), ), module.FsspInput( ENDED_DATASET, BytesIO(ended or csv_bytes("ended.csv")), "ended.csv", date(2026, 10, 5), ), ] return module.import_fssp_snapshots(inputs=inputs, load_batch=batch, **kwargs) def published(self): return list( OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE).values_list( "uid", "payload" ) ) def test_invalid_canonical_requisites_are_quarantined(self): self.org.ogrn = "bad" self.org.okpo = "1234567890" self.org.save() artifact, result = self.run_import() assert result.published == 0 assert result.quarantined == 2 assert artifact.metadata["reasons"]["invalid_canonical_requisites"] == 2 def test_two_inputs_manifest_and_stable_uid(self): artifact, result = self.run_import() assert result.parsed == 2 assert result.published == 1 assert len(artifact.metadata["inputs"]) == 2 row = OrganizationSourceRecord.objects.get(source=FSSP_SOURCE) assert row.status == "ended" assert row.amount is None assert len(row.payload["observations"]) == 2 assert row.extension.organization_id == self.org.uid uid = row.uid self.run_import(batch=2) assert OrganizationSourceRecord.objects.get(source=FSSP_SOURCE).uid == uid assert Organization.objects.count() == 1 assert row.legacy_model == "" assert row.legacy_pk == "" with pytest.raises(ValueError): _ = artifact.file.url def test_second_input_failure_preserves_snapshot(self): self.run_import() before = self.published() with pytest.raises(SnapshotValidationError): self.run_import(ended=b"invalid,csv\n", batch=2) assert self.published() == before assert ( ParserSourceArtifact.objects.filter( status=ParserSourceArtifact.Status.PUBLISHED ).count() == 1 ) def test_conflicting_duplicate_blocks_publication(self): self.run_import() before = self.published() good = csv_bytes("general.csv") changed = csv_bytes("general.csv", **{"Debt remaining balance": "100"}) rows = list(csv.reader(StringIO(changed.decode()))) handle = StringIO() csv.writer(handle).writerow(rows[1]) with pytest.raises(SnapshotValidationError, match="conflicting_fssp_identity"): self.run_import(general=good + handle.getvalue().encode(), batch=2) assert self.published() == before def test_foreign_inn_skipped_and_invalid_inn_quarantined(self): general = csv_bytes("general.csv", **{"Debtor TIN": "7704228075"}) ended = csv_bytes("ended.csv", **{"Debtor TIN": "7707083894"}) artifact, result = self.run_import(general=general, ended=ended) assert result.skipped == 1 assert result.quarantined == 1 assert result.published == 0 assert Organization.objects.count() == 1 assert artifact.metadata["reasons"]["invalid_debtor_inn"] == 1 assert artifact.staged_records.count() == 1 def test_ambiguous_identity_quarantined(self): Organization.objects.create( name="Другой филиал", inn=self.org.inn, kpp="111111111", ogrn="1027700132195", okpo="12345679", directory_imported_at=timezone.now(), ) artifact, result = self.run_import() assert result.published == 0 assert result.quarantined == 2 assert artifact.metadata["reasons"]["ambiguous_organization"] == 2 def test_missing_one_input_rejected(self): module = adapter() with pytest.raises(SnapshotValidationError, match="missing_fssp_input"): module.import_fssp_snapshots(inputs=[], load_batch=1) assert not OrganizationSourceRecord.objects.exists() def test_publication_callback_failure_rolls_back_records(self): self.run_import() before = self.published() def fail(artifact, result): raise RuntimeError("fixture publication failure") with pytest.raises(RuntimeError): self.run_import(batch=2, on_publish=fail) assert self.published() == before assert ( ParserStagedRecord.objects.filter( disposition="published", artifact__load_batch=2 ).count() == 0 ) def test_valid_empty_inputs_clear_previous_snapshot(self): self.run_import() general = csv_bytes("general.csv").split(b"\r\n", 1)[0] + b"\r\n" ended = csv_bytes("ended.csv").split(b"\r\n", 1)[0] + b"\r\n" artifact, result = self.run_import(general=general, ended=ended, batch=2) assert result.published == 0 assert not self.published() assert artifact.status == "published" def test_progress_includes_rows_outside_directory(self): rows = list( csv.reader( StringIO( csv_bytes("general.csv", **{"Debtor TIN": "7704228075"}).decode() ) ) ) handle = StringIO() csv.writer(handle).writerows([rows[0]] + [rows[1]] * 10000) progress = [] artifact, result = self.run_import( general=handle.getvalue().encode(), on_progress=lambda *counts: progress.append(counts), ) assert progress and progress[-1][0] == 10000 assert result.skipped == 10000 def test_more_than_one_staging_batch_publishes_all(self): rows = list(csv.reader(StringIO(csv_bytes("general.csv").decode()))) handle = StringIO() writer = csv.writer(handle) writer.writerow(rows[0]) number_column = rows[0].index("The number of enforcement proceeding") for number in range(501): row = list(rows[1]) row[number_column] = f"{number}/26/77001-ИП" writer.writerow(row) artifact, result = self.run_import(general=handle.getvalue().encode()) assert result.parsed == 502 assert result.published == 501 assert artifact.metadata["deduplicated_records_count"] == 1 assert artifact.staged_records.filter(disposition="published").count() == 501 def test_refresh_downloaded_artifacts_link_to_aggregate(self): module = adapter() with patch.object(module, "registry_session", return_value=DownloadSession()): artifact, result = module.refresh_fssp_enforcement_proceedings(load_batch=1) assert result.published == 1 for item in artifact.metadata["inputs"]: raw = ParserSourceArtifact.objects.get(pk=item["artifact_id"]) assert raw.metadata["aggregate_artifact_id"] == str(artifact.uid) assert raw.sha256 == item["checksum_sha256"] def test_failed_second_download_retains_first_raw_and_old_snapshot(self): self.run_import() before = self.published() module = adapter() with patch.object( module, "registry_session", return_value=DownloadSession(fail_ended=True) ), pytest.raises(requests.HTTPError): module.refresh_fssp_enforcement_proceedings(load_batch=2) assert self.published() == before raw = ParserSourceArtifact.objects.get(load_batch=2) assert raw.status == "rejected" assert raw.file.size > 0 class PassportSession: def __init__(self, body): self.body = body def get(self, url, **kwargs): response = requests.Response() response.status_code = 200 response._content = self.body response._content_consumed = True response.headers["Content-Type"] = "text/csv" return response class DownloadSession: def __init__(self, fail_ended=False): self.fail_ended = fail_ended def __enter__(self): return self def __exit__(self, *args): return None def get(self, url, **kwargs): ended = ENDED_DATASET in url if url.endswith("/meta.csv"): name = "ended-meta.csv" if ended else "general-meta.csv" return PassportSession((FIXTURES / name).read_bytes()).get(url) body = csv_bytes("ended.csv" if ended else "general.csv") response = PassportSession(body).get(url) response.status_code = 503 if ended and self.fail_ended else 206 response.headers["Content-Range"] = f"bytes 0-{len(body) - 1}/{len(body)}" response.headers["ETag"] = '"fixture"' response.url = url return response def test_passport_discovery_selects_latest_timestamp_not_row_order(): body = (FIXTURES / "general-meta.csv").read_text() lines = body.splitlines() session = PassportSession(("\n".join(reversed(lines[1:])) + "\n").encode()) with pytest.raises(SnapshotValidationError): adapter().discover_fssp_input(session, GENERAL_DATASET) session = PassportSession( ("property,value\n" + "\n".join(reversed(lines[1:]))).encode() ) url, published_at, version = adapter().discover_fssp_input(session, GENERAL_DATASET) assert url.endswith("data-20261005T0655-structure-20260820T2100.csv") assert published_at == date(2026, 10, 5) assert version == "20260820T2100" def test_passport_external_download_host_rejected(): body = ( (FIXTURES / "general-meta.csv") .read_text() .replace("https://opendata.fssp.gov.ru/", "https://foreign.example/") ) with pytest.raises(SnapshotValidationError): adapter().discover_fssp_input(PassportSession(body.encode()), GENERAL_DATASET)