fix: group JSON source exports by organization

This commit is contained in:
Aleksandr Meshchryakov
2026-09-27 10:27:06 +02:00
parent 0c4309a1bd
commit f7f81d92f5
3 changed files with 180 additions and 22 deletions

View File

@@ -8,6 +8,7 @@ import os
import re
import secrets
import shutil
import sqlite3
import zipfile
from collections.abc import Iterable, Iterator, Sequence
from dataclasses import dataclass
@@ -46,7 +47,7 @@ EXPORT_FORMAT_XLSX = "xlsx"
EXPORT_FORMAT_JSON = "json"
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
FINANCIAL_SOURCE_GROUP = "financial_indicators"
EXPORT_MANIFEST_VERSION = 2
EXPORT_MANIFEST_VERSION = 3
CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json"
GENERATION_MANIFEST_FILE_NAME = "manifest.json"
GENERATION_DIRECTORY_NAME = "generations"
@@ -630,6 +631,7 @@ def build_source_record_export_artifacts(
file_format=file_format,
)
artifact_paths = _render_source_group_artifact(
source_group=source_group,
row_spool_path=row_spool_path,
output_path=staging_directory / file_name,
headers=headers,
@@ -1048,6 +1050,7 @@ def _spool_source_group_rows(
def _render_source_group_artifact(
*,
source_group: str,
row_spool_path: Path,
output_path: Path,
headers: Sequence[str],
@@ -1068,10 +1071,87 @@ def _render_source_group_artifact(
headers=headers,
records_count=records_count,
)
os.link(row_spool_path, output_path)
_render_json_file(
source_group=source_group,
row_spool_path=row_spool_path,
output_path=output_path,
)
return (output_path,)
def _render_json_file(
*,
source_group: str,
row_spool_path: Path,
output_path: Path,
) -> None:
grouping_path = output_path.with_name(f".{output_path.name}.grouping.sqlite3")
grouping = sqlite3.connect(grouping_path)
try:
# The published spool can contain millions of unsorted records. Sort on
# local disk instead of sorting the source queryset or loading it in RAM.
grouping.execute("PRAGMA journal_mode=OFF")
grouping.execute(
"CREATE TABLE rows (group_key TEXT, position INTEGER, okpo TEXT, data TEXT)"
)
batch: list[tuple[str, int, str | None, str]] = []
for position, row in enumerate(_iter_spooled_rows(row_spool_path)):
raw_okpo = row.pop("ОКПО", None)
okpo = str(raw_okpo) if raw_okpo else None
fallback = [row.get("ИНН"), row.get("ОГРН"), row.get("Наименование")]
if not any(fallback):
fallback.append(row.get("uid"))
group_key = json.dumps(
[okpo] if okpo else [None, *fallback],
ensure_ascii=False,
)
batch.append(
(
group_key,
position,
okpo,
json.dumps(row, ensure_ascii=False, separators=(",", ":")),
)
)
if len(batch) >= SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE:
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
batch.clear()
if batch:
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
grouping.commit()
grouping.execute("CREATE INDEX rows_by_group ON rows (group_key, position)")
with output_path.open("w", encoding="utf-8") as output:
output.write("[")
current_group: str | None = None
first_record = True
for group_key, okpo, data in grouping.execute(
"SELECT group_key, okpo, data FROM rows ORDER BY group_key, position"
):
if group_key != current_group:
if current_group is not None:
output.write("]},")
output.write(
'{"okpo":'
+ json.dumps(okpo, ensure_ascii=False)
+ ","
+ json.dumps(source_group)
+ ":["
)
current_group = group_key
first_record = True
if not first_record:
output.write(",")
output.write(data)
first_record = False
if current_group is not None:
output.write("]}")
output.write("]")
finally:
grouping.close()
grouping_path.unlink(missing_ok=True)
def _iter_spooled_rows(row_spool_path: Path) -> Iterator[dict[str, Any]]:
with row_spool_path.open("r", encoding="utf-8") as rows_file:
for line in rows_file: