Files
mostovik-backend/src/apps/parsers/exchange_snapshot.py
2026-10-05 17:33:33 +02:00

255 lines
9.1 KiB
Python

"""Two private exchange inputs, one validated atomic canonical snapshot."""
from __future__ import annotations
import json
import re
from collections import Counter
from collections.abc import Callable
from io import BytesIO
from apps.parsers.exchange_moex import iter_moex_records
from apps.parsers.exchange_spb import (
EXCHANGE_SOURCE,
download_spb_artifact,
iter_spb_records,
)
from apps.parsers.fssp_csv import valid_legal_inn
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from apps.parsers.registry_snapshots import (
SNAPSHOT_CHUNK_SIZE,
OwnOrganizationIndex,
SnapshotResult,
SnapshotValidationError,
publish_snapshot,
reject_snapshot,
save_artifact_file,
stage_records,
)
def _identity(row, index):
if not valid_legal_inn(row.inn):
return None, "invalid_issuer_inn"
organization, reason = index.resolve(inn=row.inn, ogrn="")
if not reason and (
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
):
return organization, "invalid_canonical_requisites"
return organization, reason
def _stage(
rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress
):
row_number = aggregate.staged_records.count()
for row in rows:
counts["parsed"] += 1
if (
row.payload.get("exchange") != exchange
or not row.external_id.startswith(exchange + ":")
or row.record_type != "exchange_instrument"
or row.amount is not None
):
raise SnapshotValidationError("invalid_exchange_adapter_record")
organization, reason = _identity(row, index)
if reason:
counts[reason] += 1
counts["skipped" if reason == "outside_directory" else "quarantined"] += 1
if reason == "outside_directory":
continue
row.payload["provenance"] = {
**row.payload.get("provenance", {}),
"artifact_id": str(raw.uid),
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
"source_published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
}
row_number += 1
pending.append(
ParserStagedRecord(
artifact=aggregate,
row_number=row_number,
organization=organization,
external_id=row.external_id,
record_type=row.record_type,
raw_data=row.payload.get("upstream", {}),
normalized_data={
"title": row.title,
"record_date": row.record_date,
"amount": None,
"status": row.status,
"url": row.url,
"payload": row.payload,
},
disposition="quarantined" if reason else "staged",
reason=reason,
)
)
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
stage_records(pending)
if on_progress:
on_progress(
sum(c["parsed"] for c in counters.values()),
0,
aggregate.staged_records.count(),
sum(c["quarantined"] for c in counters.values()),
)
stage_records(pending)
def _save_manifest(aggregate, raws, counters):
aggregate.parsed_count = sum(c["parsed"] for c in counters.values())
aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values())
aggregate.metadata = {
**aggregate.metadata,
"inputs": [
{
"exchange": raw.metadata["dataset"],
"artifact_id": str(raw.uid),
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
"checksum_sha256": raw.sha256,
"size_bytes": raw.size_bytes,
"published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
"parsed_records_count": raw.parsed_count,
}
for raw in raws
],
"adapter_counters": {key: dict(value) for key, value in counters.items()},
"skipped_records_count": sum(c["skipped"] for c in counters.values()),
"reasons": dict(
sum(
(
Counter(
{
k: v
for k, v in c.items()
if k not in {"parsed", "skipped", "quarantined"}
}
)
for c in counters.values()
),
Counter(),
)
),
}
aggregate.rejection_reasons = aggregate.metadata["reasons"]
dates = [raw.source_published_at for raw in raws if raw.source_published_at]
aggregate.source_published_at = max(dates) if dates else None
save_artifact_file(
aggregate,
BytesIO(
json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode()
),
)
def refresh_stock_exchange_activity(
*,
load_batch: int,
uploaded_by_id: int | None = None,
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
on_progress: Callable[[int, int, int, int], None] | None = None,
) -> tuple[ParserSourceArtifact, SnapshotResult]:
aggregate = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="exchange-manifest.json",
content_type="application/json",
version="spb-csv-v1+moex-iss-v1",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
)
raws = []
counters = {"spb": Counter(), "moex": Counter()}
pending = []
try:
index = OwnOrganizationIndex()
spb = download_spb_artifact(
load_batch=load_batch, uploaded_by_id=uploaded_by_id
)
raws.append(spb)
spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)}
spb.save(update_fields=("metadata",))
with spb.file.open("rb") as handle:
_stage(
iter_spb_records(handle),
exchange="spb",
raw=spb,
aggregate=aggregate,
index=index,
counts=counters["spb"],
counters=counters,
pending=pending,
on_progress=on_progress,
)
spb.parsed_count = counters["spb"]["parsed"]
spb.status = "parsed"
spb.save()
moex = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="moex-responses.ndjson",
content_type="application/x-ndjson",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
metadata={
"dataset": "moex",
"input_only": True,
"aggregate_artifact_id": str(aggregate.uid),
},
)
raws.append(moex)
_stage(
iter_moex_records(organization_index=index, artifact=moex),
exchange="moex",
raw=moex,
aggregate=aggregate,
index=index,
counts=counters["moex"],
counters=counters,
pending=pending,
on_progress=on_progress,
)
extra = moex.metadata.get("diagnostics", {})
counters["moex"]["parsed"] = moex.parsed_count
counters["moex"]["skipped"] += extra.get("outside_directory", 0)
counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0)
counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get(
"missing_or_invalid_issuer_inn", 0
)
# Raw checksum/date are final only once the generator reaches EOF.
for staged in aggregate.staged_records.all().iterator(
chunk_size=SNAPSHOT_CHUNK_SIZE
):
raw = (
spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex
)
staged.normalized_data["payload"]["provenance"].update(
checksum_sha256=raw.sha256,
source_published_at=raw.source_published_at.isoformat()
if raw.source_published_at
else None,
)
staged.save(update_fields=("normalized_data",))
_save_manifest(aggregate, raws, counters)
aggregate.status = "parsed"
aggregate.save()
return aggregate, publish_snapshot(aggregate, on_publish=on_publish)
except Exception as exc:
# If discovery failed before returning the SPB input, retain its bounded diagnostics too.
if not raws:
raws = list(
ParserSourceArtifact.objects.filter(
source=EXCHANGE_SOURCE,
load_batch=load_batch,
metadata__input_only=True,
)
)
_save_manifest(aggregate, raws, counters)
for raw in raws:
reject_snapshot(raw, exc)
reject_snapshot(aggregate, exc)
raise