fix: group JSON source exports by organization
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Failing after 3m47s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Failing after 3m47s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped
This commit is contained in:
@@ -42,6 +42,10 @@ from rest_framework.test import APITestCase
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
|
||||
def _source_records(groups, source_group):
|
||||
return [record for group in groups for record in group[source_group]]
|
||||
|
||||
|
||||
class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
"""Checks admin-only source-record export contract."""
|
||||
|
||||
@@ -81,7 +85,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
json.dumps(
|
||||
{"uid": "record-1", "payload.rendered_page.text": value},
|
||||
ensure_ascii=False,
|
||||
) + "\n",
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
paths = _render_xlsx_files(
|
||||
@@ -91,10 +96,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
records_count=1,
|
||||
)
|
||||
workbook = load_workbook(paths[0], read_only=True)
|
||||
self.assertEqual(workbook["data"]["B2"].value, "См. overflow_values: payload.rendered_page.text")
|
||||
self.assertEqual(
|
||||
workbook["data"]["B2"].value,
|
||||
"См. overflow_values: payload.rendered_page.text",
|
||||
)
|
||||
chunks = [
|
||||
row[4][4:]
|
||||
for row in list(workbook["overflow_values_1"].values)[1:]
|
||||
row[4][4:] for row in list(workbook["overflow_values_1"].values)[1:]
|
||||
]
|
||||
self.assertEqual(base64.b64decode("".join(chunks)).decode("utf-8"), value)
|
||||
workbook.close()
|
||||
@@ -270,13 +277,24 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
rows = (
|
||||
list(csv.DictReader(StringIO(content)))
|
||||
if file_format == "csv"
|
||||
else json.loads(content)
|
||||
else _source_records(json.loads(content), "sanctions")
|
||||
)
|
||||
self.assertEqual(len(rows), 2)
|
||||
self.assertEqual({row["rn"] for row in rows}, {"0000", "0001"})
|
||||
self.assertEqual(rows[0]["inn"], "0001234567")
|
||||
self.assertNotIn("Регистрационный номер", rows[0])
|
||||
|
||||
def test_previous_json_generation_requires_rebuild(self):
|
||||
build_source_record_export_artifacts()
|
||||
manifest_path = Path(self.export_directory.name) / "current.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["version"], 3)
|
||||
manifest["version"] = 2
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
|
||||
load_current_source_record_export_generation()
|
||||
|
||||
def test_generation_contains_only_records_from_its_calendar_year(self):
|
||||
export_year = 2026
|
||||
generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC)
|
||||
@@ -356,7 +374,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
if artifact.source_group == SourceGroup.PLANNED_INSPECTIONS.value
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
inspection_rows = json.loads(inspection_path.read_text(encoding="utf-8"))
|
||||
inspection_rows = _source_records(
|
||||
json.loads(inspection_path.read_text(encoding="utf-8")),
|
||||
SourceGroup.PLANNED_INSPECTIONS.value,
|
||||
)
|
||||
self.assertEqual(
|
||||
{row["uid"] for row in inspection_rows},
|
||||
{str(current_record.pk), str(dateless_current_record.pk)},
|
||||
@@ -366,7 +387,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == SourceGroup.FINANCIAL_INDICATORS.value
|
||||
)
|
||||
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
|
||||
financial_rows = _source_records(
|
||||
json.loads(financial_path.read_text(encoding="utf-8")),
|
||||
SourceGroup.FINANCIAL_INDICATORS.value,
|
||||
)
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in financial_rows],
|
||||
[str(current_financial_record.pk)],
|
||||
@@ -408,7 +432,10 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
if artifact.source_group == SourceGroup.MEDIA_MENTIONS.value
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
rows = json.loads(media_path.read_text(encoding="utf-8"))
|
||||
rows = _source_records(
|
||||
json.loads(media_path.read_text(encoding="utf-8")),
|
||||
SourceGroup.MEDIA_MENTIONS.value,
|
||||
)
|
||||
exported = next(row for row in rows if row["uid"] == str(old_record.pk))
|
||||
self.assertEqual(exported["record_date"], "2023-02-15")
|
||||
self.assertEqual(
|
||||
@@ -529,8 +556,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
self.assertIn("payload.risk.score", rows[0])
|
||||
self.assertIn("payload.tags", rows[0])
|
||||
|
||||
financial_rows = json.loads(
|
||||
archive.read("financial-indicators.json").decode("utf-8"),
|
||||
financial_groups = json.loads(
|
||||
archive.read("financial-indicators.json").decode("utf-8")
|
||||
)
|
||||
self.assertEqual(financial_groups[0]["okpo"], "12345678")
|
||||
financial_rows = _source_records(
|
||||
financial_groups, SourceGroup.FINANCIAL_INDICATORS.value
|
||||
)
|
||||
self.assertEqual(financial_rows[0]["Наименование"], organization.full_name)
|
||||
self.assertEqual(
|
||||
@@ -803,6 +834,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
for file_format in ("json", "csv", "xlsx"):
|
||||
artifact_path = export_directory / f"artifact.{file_format}"
|
||||
_render_source_group_artifact(
|
||||
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
|
||||
row_spool_path=spool_path,
|
||||
output_path=artifact_path,
|
||||
headers=headers,
|
||||
@@ -844,18 +876,30 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
self.assertFalse(queryset.ordered)
|
||||
self.assertFalse(queryset.query.default_ordering)
|
||||
|
||||
def test_json_artifact_reuses_valid_canonical_spool_without_copying_it(self):
|
||||
def test_json_artifact_groups_nonadjacent_records_by_okpo(self):
|
||||
organization = Organization.objects.create(
|
||||
name='ООО "JSON без копии"',
|
||||
inn="7707083813",
|
||||
okpo="00123456",
|
||||
)
|
||||
other_organization = Organization.objects.create(
|
||||
name='ООО "Вторая организация"',
|
||||
inn="7707083814",
|
||||
okpo="87654321",
|
||||
)
|
||||
extension = PlannedInspectionExtension.objects.create(
|
||||
organization=organization,
|
||||
title="Плановые проверки Генпрокуратуры России",
|
||||
)
|
||||
for index in range(2):
|
||||
other_extension = PlannedInspectionExtension.objects.create(
|
||||
organization=other_organization,
|
||||
title="Плановые проверки Генпрокуратуры России",
|
||||
)
|
||||
for index, record_extension in enumerate(
|
||||
(extension, other_extension, extension)
|
||||
):
|
||||
OrganizationSourceRecord.objects.create(
|
||||
extension=extension,
|
||||
extension=record_extension,
|
||||
record_type="inspection",
|
||||
source="inspections",
|
||||
external_id=f"INSP-JSON-{index}",
|
||||
@@ -873,6 +917,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
)
|
||||
|
||||
_render_source_group_artifact(
|
||||
source_group=SourceGroup.PLANNED_INSPECTIONS.value,
|
||||
row_spool_path=spool_path,
|
||||
output_path=artifact_path,
|
||||
headers=headers,
|
||||
@@ -880,9 +925,21 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
records_count=records_count,
|
||||
)
|
||||
|
||||
self.assertEqual(records_count, 2)
|
||||
self.assertEqual(len(json.loads(spool_path.read_text())), 2)
|
||||
self.assertTrue(os.path.samefile(spool_path, artifact_path))
|
||||
self.assertEqual(records_count, 3)
|
||||
self.assertEqual(len(json.loads(spool_path.read_text())), 3)
|
||||
groups = json.loads(artifact_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
{group["okpo"] for group in groups}, {"00123456", "87654321"}
|
||||
)
|
||||
grouped = {
|
||||
group["okpo"]: group[SourceGroup.PLANNED_INSPECTIONS.value]
|
||||
for group in groups
|
||||
}
|
||||
self.assertEqual(len(grouped["00123456"]), 2)
|
||||
self.assertEqual(len(grouped["87654321"]), 1)
|
||||
self.assertNotIn("ОКПО", grouped["00123456"][0])
|
||||
self.assertFalse(os.path.samefile(spool_path, artifact_path))
|
||||
self.assertFalse(list(Path(temporary_directory).glob("*.sqlite3")))
|
||||
|
||||
def test_generation_publishes_complete_matrix_and_keeps_previous_on_failure(self):
|
||||
first_generation = build_source_record_export_artifacts()
|
||||
|
||||
Reference in New Issue
Block a user