fix: group JSON source exports by organization
Some checks failed
Mostovik Backend CI/CD / Tests and lint (pull_request) Failing after 3m47s
Mostovik Backend CI/CD / Build linux/amd64 release images (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy dev (pull_request) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (pull_request) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-09-27 10:27:06 +02:00
parent 553d787076
commit b4f3a21ad6
3 changed files with 179 additions and 27 deletions

View File

@@ -10,6 +10,7 @@ import os
import re
import secrets
import shutil
import sqlite3
import time
import zipfile
from collections.abc import Iterable, Iterator, Sequence
@@ -56,7 +57,7 @@ ALL_HISTORY_SOURCE_GROUPS = {
SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS.value,
SourceGroup.FEDRESURS_COMPANY_PROFILE.value,
}
EXPORT_MANIFEST_VERSION = 2
EXPORT_MANIFEST_VERSION = 3
CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json"
GENERATION_MANIFEST_FILE_NAME = "manifest.json"
GENERATION_DIRECTORY_NAME = "generations"
@@ -330,6 +331,7 @@ def build_source_record_export_artifacts(
artifact_paths = (output_path,)
else:
artifact_paths = _render_source_group_artifact(
source_group=source_group,
row_spool_path=row_spool_path,
output_path=staging_directory / file_name,
headers=headers,
@@ -844,6 +846,7 @@ def _spool_source_group_rows(
def _render_source_group_artifact(
*,
source_group: str,
row_spool_path: Path,
output_path: Path,
headers: Sequence[str],
@@ -866,7 +869,11 @@ def _render_source_group_artifact(
records_count=records_count,
display_headers=display_headers,
)
_render_json_file(row_spool_path=row_spool_path, output_path=output_path)
_render_json_file(
source_group=source_group,
row_spool_path=row_spool_path,
output_path=output_path,
)
return (output_path,)
@@ -1156,10 +1163,79 @@ def _build_xlsx_part_path(
def _render_json_file(
*,
source_group: str,
row_spool_path: Path,
output_path: Path,
) -> None:
os.link(row_spool_path, output_path)
grouping_path = output_path.with_name(f".{output_path.name}.grouping.sqlite3")
grouping = sqlite3.connect(grouping_path)
try:
# The published spool can contain millions of unsorted records. Sort on
# local disk instead of sorting the source queryset or loading it in RAM.
grouping.execute("PRAGMA journal_mode=OFF")
grouping.execute(
"CREATE TABLE rows (group_key TEXT, position INTEGER, okpo TEXT, data TEXT)"
)
batch: list[tuple[str, int, str | None, str]] = []
for position, row in enumerate(_iter_spooled_rows(row_spool_path)):
raw_okpo = row.pop("ОКПО", None) or row.pop("okpo", None)
okpo = str(raw_okpo) if raw_okpo else None
fallback = [
row.get("ИНН") or row.get("inn"),
row.get("ОГРН") or row.get("ogrn"),
row.get("Наименование"),
]
if not any(fallback):
fallback.append(row.get("uid"))
group_key = json.dumps(
[okpo] if okpo else [None, *fallback],
ensure_ascii=False,
)
batch.append(
(
group_key,
position,
okpo,
json.dumps(row, ensure_ascii=False, separators=(",", ":")),
)
)
if len(batch) >= SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE:
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
batch.clear()
if batch:
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
grouping.commit()
grouping.execute("CREATE INDEX rows_by_group ON rows (group_key, position)")
with output_path.open("w", encoding="utf-8") as output:
output.write("[")
current_group: str | None = None
first_record = True
for group_key, okpo, data in grouping.execute(
"SELECT group_key, okpo, data FROM rows ORDER BY group_key, position"
):
if group_key != current_group:
if current_group is not None:
output.write("]},")
output.write(
'{"okpo":'
+ json.dumps(okpo, ensure_ascii=False)
+ ","
+ json.dumps(source_group)
+ ":["
)
current_group = group_key
first_record = True
if not first_record:
output.write(",")
output.write(data)
first_record = False
if current_group is not None:
output.write("]}")
output.write("]")
finally:
grouping.close()
grouping_path.unlink(missing_ok=True)
def _build_record_row(