310 lines
12 KiB
Python
310 lines
12 KiB
Python
"""Atomic FSSP publication: real staging/database, synthetic input files."""
|
||
|
||
import csv
|
||
import importlib
|
||
from datetime import date
|
||
from io import BytesIO, StringIO
|
||
from pathlib import Path
|
||
from tempfile import TemporaryDirectory
|
||
from unittest.mock import patch
|
||
|
||
import pytest
|
||
import requests
|
||
from apps.parsers.fssp_csv import ENDED_DATASET, FSSP_SOURCE, GENERAL_DATASET
|
||
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||
from django.test import TestCase, override_settings
|
||
from django.utils import timezone
|
||
from organizations.models import Organization, OrganizationSourceRecord
|
||
|
||
FIXTURES = Path(__file__).parents[2] / "fixtures" / "fssp"
|
||
|
||
|
||
def adapter():
|
||
try:
|
||
return importlib.import_module("apps.parsers.fssp_snapshot")
|
||
except ModuleNotFoundError:
|
||
pytest.fail("FSSP snapshot importer is not implemented")
|
||
|
||
|
||
def csv_bytes(name, **changes):
|
||
rows = list(csv.reader(StringIO((FIXTURES / name).read_text())))
|
||
for key, value in changes.items():
|
||
rows[1][rows[0].index(key)] = value
|
||
handle = StringIO()
|
||
csv.writer(handle).writerows(rows)
|
||
return handle.getvalue().encode()
|
||
|
||
|
||
class FsspSnapshotTests(TestCase):
|
||
def setUp(self):
|
||
self.tmp = TemporaryDirectory()
|
||
self.addCleanup(self.tmp.cleanup)
|
||
self.settings = override_settings(
|
||
MEDIA_ROOT=self.tmp.name + "/public",
|
||
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
|
||
)
|
||
self.settings.enable()
|
||
self.addCleanup(self.settings.disable)
|
||
self.org = Organization.objects.create(
|
||
name="АО Проверка",
|
||
inn="7707083893",
|
||
ogrn="1027700132195",
|
||
okpo="12345678",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
|
||
def run_import(self, general=None, ended=None, batch=1, **kwargs):
|
||
module = adapter()
|
||
inputs = [
|
||
module.FsspInput(
|
||
GENERAL_DATASET,
|
||
BytesIO(general or csv_bytes("general.csv")),
|
||
"general.csv",
|
||
date(2026, 10, 5),
|
||
),
|
||
module.FsspInput(
|
||
ENDED_DATASET,
|
||
BytesIO(ended or csv_bytes("ended.csv")),
|
||
"ended.csv",
|
||
date(2026, 10, 5),
|
||
),
|
||
]
|
||
return module.import_fssp_snapshots(inputs=inputs, load_batch=batch, **kwargs)
|
||
|
||
def published(self):
|
||
return list(
|
||
OrganizationSourceRecord.objects.filter(source=FSSP_SOURCE).values_list(
|
||
"uid", "payload"
|
||
)
|
||
)
|
||
|
||
def test_invalid_canonical_requisites_are_quarantined(self):
|
||
self.org.ogrn = "bad"
|
||
self.org.okpo = "1234567890"
|
||
self.org.save()
|
||
artifact, result = self.run_import()
|
||
assert result.published == 0
|
||
assert result.quarantined == 2
|
||
assert artifact.metadata["reasons"]["invalid_canonical_requisites"] == 2
|
||
|
||
def test_two_inputs_manifest_and_stable_uid(self):
|
||
artifact, result = self.run_import()
|
||
assert result.parsed == 2
|
||
assert result.published == 1
|
||
assert len(artifact.metadata["inputs"]) == 2
|
||
row = OrganizationSourceRecord.objects.get(source=FSSP_SOURCE)
|
||
assert row.status == "ended"
|
||
assert row.amount is None
|
||
assert len(row.payload["observations"]) == 2
|
||
assert row.extension.organization_id == self.org.uid
|
||
uid = row.uid
|
||
self.run_import(batch=2)
|
||
assert OrganizationSourceRecord.objects.get(source=FSSP_SOURCE).uid == uid
|
||
assert Organization.objects.count() == 1
|
||
assert row.legacy_model == ""
|
||
assert row.legacy_pk == ""
|
||
with pytest.raises(ValueError):
|
||
_ = artifact.file.url
|
||
|
||
def test_second_input_failure_preserves_snapshot(self):
|
||
self.run_import()
|
||
before = self.published()
|
||
with pytest.raises(SnapshotValidationError):
|
||
self.run_import(ended=b"invalid,csv\n", batch=2)
|
||
assert self.published() == before
|
||
assert (
|
||
ParserSourceArtifact.objects.filter(
|
||
status=ParserSourceArtifact.Status.PUBLISHED
|
||
).count()
|
||
== 1
|
||
)
|
||
|
||
def test_conflicting_duplicate_blocks_publication(self):
|
||
self.run_import()
|
||
before = self.published()
|
||
good = csv_bytes("general.csv")
|
||
changed = csv_bytes("general.csv", **{"Debt remaining balance": "100"})
|
||
rows = list(csv.reader(StringIO(changed.decode())))
|
||
handle = StringIO()
|
||
csv.writer(handle).writerow(rows[1])
|
||
with pytest.raises(SnapshotValidationError, match="conflicting_fssp_identity"):
|
||
self.run_import(general=good + handle.getvalue().encode(), batch=2)
|
||
assert self.published() == before
|
||
|
||
def test_foreign_inn_skipped_and_invalid_inn_quarantined(self):
|
||
general = csv_bytes("general.csv", **{"Debtor TIN": "7704228075"})
|
||
ended = csv_bytes("ended.csv", **{"Debtor TIN": "7707083894"})
|
||
artifact, result = self.run_import(general=general, ended=ended)
|
||
assert result.skipped == 1
|
||
assert result.quarantined == 1
|
||
assert result.published == 0
|
||
assert Organization.objects.count() == 1
|
||
assert artifact.metadata["reasons"]["invalid_debtor_inn"] == 1
|
||
assert artifact.staged_records.count() == 1
|
||
|
||
def test_ambiguous_identity_quarantined(self):
|
||
Organization.objects.create(
|
||
name="Другой филиал",
|
||
inn=self.org.inn,
|
||
kpp="111111111",
|
||
ogrn="1027700132195",
|
||
okpo="12345679",
|
||
directory_imported_at=timezone.now(),
|
||
)
|
||
artifact, result = self.run_import()
|
||
assert result.published == 0
|
||
assert result.quarantined == 2
|
||
assert artifact.metadata["reasons"]["ambiguous_organization"] == 2
|
||
|
||
def test_missing_one_input_rejected(self):
|
||
module = adapter()
|
||
with pytest.raises(SnapshotValidationError, match="missing_fssp_input"):
|
||
module.import_fssp_snapshots(inputs=[], load_batch=1)
|
||
assert not OrganizationSourceRecord.objects.exists()
|
||
|
||
def test_publication_callback_failure_rolls_back_records(self):
|
||
self.run_import()
|
||
before = self.published()
|
||
|
||
def fail(artifact, result):
|
||
raise RuntimeError("fixture publication failure")
|
||
|
||
with pytest.raises(RuntimeError):
|
||
self.run_import(batch=2, on_publish=fail)
|
||
assert self.published() == before
|
||
assert (
|
||
ParserStagedRecord.objects.filter(
|
||
disposition="published", artifact__load_batch=2
|
||
).count()
|
||
== 0
|
||
)
|
||
|
||
def test_valid_empty_inputs_clear_previous_snapshot(self):
|
||
self.run_import()
|
||
general = csv_bytes("general.csv").split(b"\r\n", 1)[0] + b"\r\n"
|
||
ended = csv_bytes("ended.csv").split(b"\r\n", 1)[0] + b"\r\n"
|
||
artifact, result = self.run_import(general=general, ended=ended, batch=2)
|
||
assert result.published == 0
|
||
assert not self.published()
|
||
assert artifact.status == "published"
|
||
|
||
def test_progress_includes_rows_outside_directory(self):
|
||
rows = list(
|
||
csv.reader(
|
||
StringIO(
|
||
csv_bytes("general.csv", **{"Debtor TIN": "7704228075"}).decode()
|
||
)
|
||
)
|
||
)
|
||
handle = StringIO()
|
||
csv.writer(handle).writerows([rows[0]] + [rows[1]] * 10000)
|
||
progress = []
|
||
artifact, result = self.run_import(
|
||
general=handle.getvalue().encode(),
|
||
on_progress=lambda *counts: progress.append(counts),
|
||
)
|
||
assert progress and progress[-1][0] == 10000
|
||
assert result.skipped == 10000
|
||
|
||
def test_more_than_one_staging_batch_publishes_all(self):
|
||
rows = list(csv.reader(StringIO(csv_bytes("general.csv").decode())))
|
||
handle = StringIO()
|
||
writer = csv.writer(handle)
|
||
writer.writerow(rows[0])
|
||
number_column = rows[0].index("The number of enforcement proceeding")
|
||
for number in range(501):
|
||
row = list(rows[1])
|
||
row[number_column] = f"{number}/26/77001-ИП"
|
||
writer.writerow(row)
|
||
artifact, result = self.run_import(general=handle.getvalue().encode())
|
||
assert result.parsed == 502
|
||
assert result.published == 501
|
||
assert artifact.metadata["deduplicated_records_count"] == 1
|
||
assert artifact.staged_records.filter(disposition="published").count() == 501
|
||
|
||
def test_refresh_downloaded_artifacts_link_to_aggregate(self):
|
||
module = adapter()
|
||
with patch.object(module, "registry_session", return_value=DownloadSession()):
|
||
artifact, result = module.refresh_fssp_enforcement_proceedings(load_batch=1)
|
||
assert result.published == 1
|
||
for item in artifact.metadata["inputs"]:
|
||
raw = ParserSourceArtifact.objects.get(pk=item["artifact_id"])
|
||
assert raw.metadata["aggregate_artifact_id"] == str(artifact.uid)
|
||
assert raw.sha256 == item["checksum_sha256"]
|
||
|
||
def test_failed_second_download_retains_first_raw_and_old_snapshot(self):
|
||
self.run_import()
|
||
before = self.published()
|
||
module = adapter()
|
||
with patch.object(
|
||
module, "registry_session", return_value=DownloadSession(fail_ended=True)
|
||
), pytest.raises(requests.HTTPError):
|
||
module.refresh_fssp_enforcement_proceedings(load_batch=2)
|
||
assert self.published() == before
|
||
raw = ParserSourceArtifact.objects.get(load_batch=2)
|
||
assert raw.status == "rejected"
|
||
assert raw.file.size > 0
|
||
|
||
|
||
class PassportSession:
|
||
def __init__(self, body):
|
||
self.body = body
|
||
|
||
def get(self, url, **kwargs):
|
||
response = requests.Response()
|
||
response.status_code = 200
|
||
response._content = self.body
|
||
response._content_consumed = True
|
||
response.headers["Content-Type"] = "text/csv"
|
||
return response
|
||
|
||
|
||
class DownloadSession:
|
||
def __init__(self, fail_ended=False):
|
||
self.fail_ended = fail_ended
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, *args):
|
||
return None
|
||
|
||
def get(self, url, **kwargs):
|
||
ended = ENDED_DATASET in url
|
||
if url.endswith("/meta.csv"):
|
||
name = "ended-meta.csv" if ended else "general-meta.csv"
|
||
return PassportSession((FIXTURES / name).read_bytes()).get(url)
|
||
body = csv_bytes("ended.csv" if ended else "general.csv")
|
||
response = PassportSession(body).get(url)
|
||
response.status_code = 503 if ended and self.fail_ended else 206
|
||
response.headers["Content-Range"] = f"bytes 0-{len(body) - 1}/{len(body)}"
|
||
response.headers["ETag"] = '"fixture"'
|
||
response.url = url
|
||
return response
|
||
|
||
|
||
def test_passport_discovery_selects_latest_timestamp_not_row_order():
|
||
body = (FIXTURES / "general-meta.csv").read_text()
|
||
lines = body.splitlines()
|
||
session = PassportSession(("\n".join(reversed(lines[1:])) + "\n").encode())
|
||
with pytest.raises(SnapshotValidationError):
|
||
adapter().discover_fssp_input(session, GENERAL_DATASET)
|
||
session = PassportSession(
|
||
("property,value\n" + "\n".join(reversed(lines[1:]))).encode()
|
||
)
|
||
url, published_at, version = adapter().discover_fssp_input(session, GENERAL_DATASET)
|
||
assert url.endswith("data-20261005T0655-structure-20260820T2100.csv")
|
||
assert published_at == date(2026, 10, 5)
|
||
assert version == "20260820T2100"
|
||
|
||
|
||
def test_passport_external_download_host_rejected():
|
||
body = (
|
||
(FIXTURES / "general-meta.csv")
|
||
.read_text()
|
||
.replace("https://opendata.fssp.gov.ru/", "https://foreign.example/")
|
||
)
|
||
with pytest.raises(SnapshotValidationError):
|
||
adapter().discover_fssp_input(PassportSession(body.encode()), GENERAL_DATASET)
|