fix: group JSON source exports by organization
This commit is contained in:
@@ -8,6 +8,7 @@ import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import sqlite3
|
||||
import zipfile
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
@@ -46,7 +47,7 @@ EXPORT_FORMAT_XLSX = "xlsx"
|
||||
EXPORT_FORMAT_JSON = "json"
|
||||
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
|
||||
FINANCIAL_SOURCE_GROUP = "financial_indicators"
|
||||
EXPORT_MANIFEST_VERSION = 2
|
||||
EXPORT_MANIFEST_VERSION = 3
|
||||
CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json"
|
||||
GENERATION_MANIFEST_FILE_NAME = "manifest.json"
|
||||
GENERATION_DIRECTORY_NAME = "generations"
|
||||
@@ -630,6 +631,7 @@ def build_source_record_export_artifacts(
|
||||
file_format=file_format,
|
||||
)
|
||||
artifact_paths = _render_source_group_artifact(
|
||||
source_group=source_group,
|
||||
row_spool_path=row_spool_path,
|
||||
output_path=staging_directory / file_name,
|
||||
headers=headers,
|
||||
@@ -1048,6 +1050,7 @@ def _spool_source_group_rows(
|
||||
|
||||
def _render_source_group_artifact(
|
||||
*,
|
||||
source_group: str,
|
||||
row_spool_path: Path,
|
||||
output_path: Path,
|
||||
headers: Sequence[str],
|
||||
@@ -1068,10 +1071,87 @@ def _render_source_group_artifact(
|
||||
headers=headers,
|
||||
records_count=records_count,
|
||||
)
|
||||
os.link(row_spool_path, output_path)
|
||||
_render_json_file(
|
||||
source_group=source_group,
|
||||
row_spool_path=row_spool_path,
|
||||
output_path=output_path,
|
||||
)
|
||||
return (output_path,)
|
||||
|
||||
|
||||
def _render_json_file(
|
||||
*,
|
||||
source_group: str,
|
||||
row_spool_path: Path,
|
||||
output_path: Path,
|
||||
) -> None:
|
||||
grouping_path = output_path.with_name(f".{output_path.name}.grouping.sqlite3")
|
||||
grouping = sqlite3.connect(grouping_path)
|
||||
try:
|
||||
# The published spool can contain millions of unsorted records. Sort on
|
||||
# local disk instead of sorting the source queryset or loading it in RAM.
|
||||
grouping.execute("PRAGMA journal_mode=OFF")
|
||||
grouping.execute(
|
||||
"CREATE TABLE rows (group_key TEXT, position INTEGER, okpo TEXT, data TEXT)"
|
||||
)
|
||||
batch: list[tuple[str, int, str | None, str]] = []
|
||||
for position, row in enumerate(_iter_spooled_rows(row_spool_path)):
|
||||
raw_okpo = row.pop("ОКПО", None)
|
||||
okpo = str(raw_okpo) if raw_okpo else None
|
||||
fallback = [row.get("ИНН"), row.get("ОГРН"), row.get("Наименование")]
|
||||
if not any(fallback):
|
||||
fallback.append(row.get("uid"))
|
||||
group_key = json.dumps(
|
||||
[okpo] if okpo else [None, *fallback],
|
||||
ensure_ascii=False,
|
||||
)
|
||||
batch.append(
|
||||
(
|
||||
group_key,
|
||||
position,
|
||||
okpo,
|
||||
json.dumps(row, ensure_ascii=False, separators=(",", ":")),
|
||||
)
|
||||
)
|
||||
if len(batch) >= SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE:
|
||||
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
|
||||
batch.clear()
|
||||
if batch:
|
||||
grouping.executemany("INSERT INTO rows VALUES (?, ?, ?, ?)", batch)
|
||||
grouping.commit()
|
||||
grouping.execute("CREATE INDEX rows_by_group ON rows (group_key, position)")
|
||||
|
||||
with output_path.open("w", encoding="utf-8") as output:
|
||||
output.write("[")
|
||||
current_group: str | None = None
|
||||
first_record = True
|
||||
for group_key, okpo, data in grouping.execute(
|
||||
"SELECT group_key, okpo, data FROM rows ORDER BY group_key, position"
|
||||
):
|
||||
if group_key != current_group:
|
||||
if current_group is not None:
|
||||
output.write("]},")
|
||||
output.write(
|
||||
'{"okpo":'
|
||||
+ json.dumps(okpo, ensure_ascii=False)
|
||||
+ ","
|
||||
+ json.dumps(source_group)
|
||||
+ ":["
|
||||
)
|
||||
current_group = group_key
|
||||
first_record = True
|
||||
if not first_record:
|
||||
output.write(",")
|
||||
output.write(data)
|
||||
first_record = False
|
||||
if current_group is not None:
|
||||
output.write("]}")
|
||||
output.write("]")
|
||||
finally:
|
||||
grouping.close()
|
||||
grouping_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _iter_spooled_rows(row_spool_path: Path) -> Iterator[dict[str, Any]]:
|
||||
with row_spool_path.open("r", encoding="utf-8") as rows_file:
|
||||
for line in rows_file:
|
||||
|
||||
Reference in New Issue
Block a user