fix: group JSON source exports by organization
This commit is contained in:
@@ -4,6 +4,7 @@ import json
|
||||
import zipfile
|
||||
from datetime import UTC, datetime
|
||||
from io import BytesIO, StringIO
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -13,6 +14,7 @@ from apps.external_data.source_record_export import (
|
||||
SOURCE_GROUP_EXPORT_SPECS,
|
||||
SOURCE_RECORD_EXPORT_FIELDS,
|
||||
SourceRecordExportArtifactsUnavailable,
|
||||
_render_json_file,
|
||||
_source_group_headers,
|
||||
build_source_record_export_artifacts,
|
||||
build_source_records_export_archive,
|
||||
@@ -40,6 +42,40 @@ from tests.apps.organization.factories import OrganizationFactory
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
|
||||
def _source_records(groups, source_group):
|
||||
return [record for group in groups for record in group[source_group]]
|
||||
|
||||
|
||||
def test_json_groups_interleaved_rows_by_okpo():
|
||||
rows = [
|
||||
{"ОКПО": "00123456", "ИНН": "111", "uid": "first"},
|
||||
{"ОКПО": "87654321", "ИНН": "222", "uid": "other"},
|
||||
{"ОКПО": "00123456", "ИНН": "111", "uid": "last"},
|
||||
]
|
||||
with TemporaryDirectory() as directory:
|
||||
spool_path = Path(directory) / "rows.json"
|
||||
output_path = Path(directory) / "output.json"
|
||||
spool_path.write_text(
|
||||
"[\n" + ",\n".join(json.dumps(row) for row in rows) + "\n]",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
_render_json_file(
|
||||
source_group="bankruptcy",
|
||||
row_spool_path=spool_path,
|
||||
output_path=output_path,
|
||||
)
|
||||
|
||||
groups = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
self_groups = {group["okpo"]: group["bankruptcy"] for group in groups}
|
||||
assert [row["uid"] for row in self_groups["00123456"]] == ["first", "last"]
|
||||
assert [row["uid"] for row in self_groups["87654321"]] == ["other"]
|
||||
assert all(
|
||||
"ОКПО" not in record for record in _source_records(groups, "bankruptcy")
|
||||
)
|
||||
assert not list(Path(directory).glob("*.sqlite3"))
|
||||
|
||||
|
||||
class SourceRecordExportApiTest(APITestCase):
|
||||
"""Check admin access and zero-query delivery of prepared files."""
|
||||
|
||||
@@ -122,7 +158,8 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
if artifact.source_group == "industrial_production"
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
industrial_rows = json.loads(industrial_path.read_text(encoding="utf-8"))
|
||||
industrial_groups = json.loads(industrial_path.read_text(encoding="utf-8"))
|
||||
industrial_rows = _source_records(industrial_groups, "industrial_production")
|
||||
self.assertEqual(
|
||||
{row["record_type"] for row in industrial_rows},
|
||||
{
|
||||
@@ -131,11 +168,8 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
"manufacturer_registry_entry",
|
||||
},
|
||||
)
|
||||
self.assertEqual({row["ОКПО"] for row in industrial_rows}, {"12345678"})
|
||||
self.assertEqual(
|
||||
list(industrial_rows[0])[: len(expected_prefix)],
|
||||
expected_prefix,
|
||||
)
|
||||
self.assertEqual([group["okpo"] for group in industrial_groups], ["12345678"])
|
||||
self.assertNotIn("ОКПО", industrial_rows[0])
|
||||
self.assertNotIn("organization", industrial_rows[0])
|
||||
self.assertIn("payload.product_name", industrial_rows[0])
|
||||
|
||||
@@ -144,7 +178,10 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == "financial_indicators"
|
||||
)
|
||||
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
|
||||
financial_rows = _source_records(
|
||||
json.loads(financial_path.read_text(encoding="utf-8")),
|
||||
"financial_indicators",
|
||||
)
|
||||
self.assertEqual(financial_rows[0]["financial_lines"][0]["line_code"], "1600")
|
||||
|
||||
arbitration_path = next(
|
||||
@@ -152,7 +189,9 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == "arbitration" and artifact.file_format == "json"
|
||||
)
|
||||
arbitration_row = json.loads(arbitration_path.read_text(encoding="utf-8"))[0]
|
||||
arbitration_row = _source_records(
|
||||
json.loads(arbitration_path.read_text(encoding="utf-8")), "arbitration"
|
||||
)[0]
|
||||
self.assertEqual(arbitration_row["source"], "arbitration")
|
||||
self.assertEqual(
|
||||
arbitration_row["external_id"], arbitration_row["payload.case_number"]
|
||||
@@ -252,7 +291,9 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
)
|
||||
for artifact in artifacts:
|
||||
if artifact.file_format == "json":
|
||||
exported_values = json.loads(artifact.path.read_text(encoding="utf-8"))
|
||||
groups = json.loads(artifact.path.read_text(encoding="utf-8"))
|
||||
self.assertEqual([group["okpo"] for group in groups], ["11223344"])
|
||||
exported_values = _source_records(groups, "bankruptcy")
|
||||
self.assertEqual(
|
||||
[key for key in exported_values[0] if key.startswith("payload.")],
|
||||
[
|
||||
@@ -277,7 +318,8 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
serialized_values = str(exported_values)
|
||||
self.assertIn(str(excluded_record.id), serialized_values)
|
||||
self.assertIn(str(included_record.id), str(exported_values))
|
||||
self.assertIn("11223344", str(exported_values))
|
||||
if artifact.file_format != "json":
|
||||
self.assertIn("11223344", str(exported_values))
|
||||
self.assertNotRegex(
|
||||
serialized_values,
|
||||
r"(?i)(?<![a-zа-яё0-9])(?:checko(?:\.ru)?|чеко|чекало|чекко)"
|
||||
@@ -294,6 +336,17 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
self.assertEqual(generation.export_year, timezone.localdate().year)
|
||||
self.assertIn('"artifacts_count": 31', command_output.getvalue())
|
||||
|
||||
def test_previous_json_generation_requires_rebuild(self):
|
||||
build_source_record_export_artifacts()
|
||||
manifest_path = Path(self.export_directory.name) / "current.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(manifest["version"], 3)
|
||||
manifest["version"] = 2
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
|
||||
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
|
||||
load_current_source_record_export_generation()
|
||||
|
||||
def test_generation_contains_only_records_from_its_calendar_year(self):
|
||||
export_year = 2026
|
||||
generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC)
|
||||
@@ -334,7 +387,10 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
if artifact.source_group == "government_procurements"
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
procurement_rows = json.loads(procurements_path.read_text(encoding="utf-8"))
|
||||
procurement_rows = _source_records(
|
||||
json.loads(procurements_path.read_text(encoding="utf-8")),
|
||||
"government_procurements",
|
||||
)
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in procurement_rows],
|
||||
[str(current_procurement.pk)],
|
||||
@@ -346,7 +402,10 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
if artifact.source_group == "industrial_production"
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
industrial_rows = json.loads(industrial_path.read_text(encoding="utf-8"))
|
||||
industrial_rows = _source_records(
|
||||
json.loads(industrial_path.read_text(encoding="utf-8")),
|
||||
"industrial_production",
|
||||
)
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in industrial_rows],
|
||||
[str(current_product.pk)],
|
||||
@@ -357,7 +416,10 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == "financial_indicators"
|
||||
)
|
||||
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
|
||||
financial_rows = _source_records(
|
||||
json.loads(financial_path.read_text(encoding="utf-8")),
|
||||
"financial_indicators",
|
||||
)
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in financial_rows],
|
||||
[str(current_report.pk)],
|
||||
|
||||
Reference in New Issue
Block a user