fix: group JSON source exports by organization
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Failing after 3m47s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-09-27 10:27:06 +02:00
parent 553d787076
commit b4f3a21ad6
3 changed files with 179 additions and 27 deletions

View File

@@ -42,6 +42,10 @@ from rest_framework.test import APITestCase
from tests.apps.user.factories import UserFactory
def _source_records(groups, source_group):
return [record for group in groups for record in group[source_group]]
class OrganizationSourceRecordExportApiV2Test(APITestCase):
"""Checks admin-only source-record export contract."""
@@ -81,7 +85,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
json.dumps(
{"uid": "record-1", "payload.rendered_page.text": value},
ensure_ascii=False,
) + "\n",
)
+ "\n",
encoding="utf-8",
)
paths = _render_xlsx_files(
@@ -91,10 +96,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
records_count=1,
)
workbook = load_workbook(paths[0], read_only=True)
self.assertEqual(workbook["data"]["B2"].value, "См. overflow_values: payload.rendered_page.text")
self.assertEqual(
workbook["data"]["B2"].value,
"См. overflow_values: payload.rendered_page.text",
)
chunks = [
row[4][4:]
for row in list(workbook["overflow_values_1"].values)[1:]
row[4][4:] for row in list(workbook["overflow_values_1"].values)[1:]
]
self.assertEqual(base64.b64decode("".join(chunks)).decode("utf-8"), value)
workbook.close()
@@ -270,13 +277,24 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
rows = (
list(csv.DictReader(StringIO(content)))
if file_format == "csv"
else json.loads(content)
else _source_records(json.loads(content), "sanctions")
)
self.assertEqual(len(rows), 2)
self.assertEqual({row["rn"] for row in rows}, {"0000", "0001"})
self.assertEqual(rows[0]["inn"], "0001234567")
self.assertNotIn("Регистрационный номер", rows[0])
def test_previous_json_generation_requires_rebuild(self):
build_source_record_export_artifacts()
manifest_path = Path(self.export_directory.name) / "current.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
self.assertEqual(manifest["version"], 3)
manifest["version"] = 2
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
load_current_source_record_export_generation()
def test_generation_contains_only_records_from_its_calendar_year(self):
export_year = 2026
generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC)
@@ -356,7 +374,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
if artifact.source_group == SourceGroup.PLANNED_INSPECTIONS.value
and artifact.file_format == "json"
)
inspection_rows = json.loads(inspection_path.read_text(encoding="utf-8"))
inspection_rows = _source_records(
json.loads(inspection_path.read_text(encoding="utf-8")),
SourceGroup.PLANNED_INSPECTIONS.value,
)
self.assertEqual(
{row["uid"] for row in inspection_rows},
{str(current_record.pk), str(dateless_current_record.pk)},
@@ -366,7 +387,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
for artifact in generation.artifacts
if artifact.source_group == SourceGroup.FINANCIAL_INDICATORS.value
)
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
financial_rows = _source_records(
json.loads(financial_path.read_text(encoding="utf-8")),
SourceGroup.FINANCIAL_INDICATORS.value,
)
self.assertEqual(
[row["uid"] for row in financial_rows],
[str(current_financial_record.pk)],
@@ -408,7 +432,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
if artifact.source_group == SourceGroup.MEDIA_MENTIONS.value
and artifact.file_format == "json"
)
rows = json.loads(media_path.read_text(encoding="utf-8"))
rows = _source_records(
json.loads(media_path.read_text(encoding="utf-8")),
SourceGroup.MEDIA_MENTIONS.value,
)
exported = next(row for row in rows if row["uid"] == str(old_record.pk))
self.assertEqual(exported["record_date"], "2023-02-15")
self.assertEqual(
@@ -529,8 +556,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertIn("payload.risk.score", rows[0])
self.assertIn("payload.tags", rows[0])
financial_rows = json.loads(
archive.read("financial-indicators.json").decode("utf-8"),
financial_groups = json.loads(
archive.read("financial-indicators.json").decode("utf-8")
)
self.assertEqual(financial_groups[0]["okpo"], "12345678")
financial_rows = _source_records(
financial_groups, SourceGroup.FINANCIAL_INDICATORS.value
)
self.assertEqual(financial_rows[0]["Наименование"], organization.full_name)
self.assertEqual(
@@ -803,6 +834,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
for file_format in ("json", "csv", "xlsx"):
artifact_path = export_directory / f"artifact.{file_format}"
_render_source_group_artifact(
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
row_spool_path=spool_path,
output_path=artifact_path,
headers=headers,
@@ -844,18 +876,30 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertFalse(queryset.ordered)
self.assertFalse(queryset.query.default_ordering)
def test_json_artifact_reuses_valid_canonical_spool_without_copying_it(self):
def test_json_artifact_groups_nonadjacent_records_by_okpo(self):
organization = Organization.objects.create(
name='ООО "JSON без копии"',
inn="7707083813",
okpo="00123456",
)
other_organization = Organization.objects.create(
name='ООО "Вторая организация"',
inn="7707083814",
okpo="87654321",
)
extension = PlannedInspectionExtension.objects.create(
organization=organization,
title="Плановые проверки Генпрокуратуры России",
)
for index in range(2):
other_extension = PlannedInspectionExtension.objects.create(
organization=other_organization,
title="Плановые проверки Генпрокуратуры России",
)
for index, record_extension in enumerate(
(extension, other_extension, extension)
):
OrganizationSourceRecord.objects.create(
extension=extension,
extension=record_extension,
record_type="inspection",
source="inspections",
external_id=f"INSP-JSON-{index}",
@@ -873,6 +917,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
)
_render_source_group_artifact(
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
row_spool_path=spool_path,
output_path=artifact_path,
headers=headers,
@@ -880,9 +925,21 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
records_count=records_count,
)
self.assertEqual(records_count, 2)
self.assertEqual(len(json.loads(spool_path.read_text())), 2)
self.assertTrue(os.path.samefile(spool_path, artifact_path))
self.assertEqual(records_count, 3)
self.assertEqual(len(json.loads(spool_path.read_text())), 3)
groups = json.loads(artifact_path.read_text(encoding="utf-8"))
self.assertEqual(
{group["okpo"] for group in groups}, {"00123456", "87654321"}
)
grouped = {
group["okpo"]: group[SourceGroup.PLANNED_INSPECTIONS.value]
for group in groups
}
self.assertEqual(len(grouped["00123456"]), 2)
self.assertEqual(len(grouped["87654321"]), 1)
self.assertNotIn("ОКПО", grouped["00123456"][0])
self.assertFalse(os.path.samefile(spool_path, artifact_path))
self.assertFalse(list(Path(temporary_directory).glob("*.sqlite3")))
def test_generation_publishes_complete_matrix_and_keeps_previous_on_failure(self):
first_generation = build_source_record_export_artifacts()