feat: publish unified exchange activity snapshots
This commit is contained in:
252
src/apps/parsers/exchange_snapshot.py
Normal file
252
src/apps/parsers/exchange_snapshot.py
Normal file
@@ -0,0 +1,252 @@
|
||||
"""Two private exchange inputs, one validated atomic canonical snapshot."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from io import BytesIO
|
||||
|
||||
from apps.parsers.exchange_moex import iter_moex_records
|
||||
from apps.parsers.exchange_spb import (
|
||||
EXCHANGE_SOURCE,
|
||||
download_spb_artifact,
|
||||
iter_spb_records,
|
||||
)
|
||||
from apps.parsers.fssp_csv import valid_legal_inn
|
||||
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
||||
from apps.parsers.registry_snapshots import (
|
||||
SNAPSHOT_CHUNK_SIZE,
|
||||
OwnOrganizationIndex,
|
||||
SnapshotResult,
|
||||
SnapshotValidationError,
|
||||
publish_snapshot,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
stage_records,
|
||||
)
|
||||
|
||||
|
||||
def _identity(row, index):
|
||||
if not valid_legal_inn(row.inn):
|
||||
return None, "invalid_issuer_inn"
|
||||
organization, reason = index.resolve(inn=row.inn, ogrn="")
|
||||
if not reason and (
|
||||
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
|
||||
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
|
||||
):
|
||||
return organization, "invalid_canonical_requisites"
|
||||
return organization, reason
|
||||
|
||||
|
||||
def _stage(
|
||||
rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress
|
||||
):
|
||||
row_number = aggregate.staged_records.count()
|
||||
for row in rows:
|
||||
counts["parsed"] += 1
|
||||
if (
|
||||
row.payload.get("exchange") != exchange
|
||||
or not row.external_id.startswith(exchange + ":")
|
||||
or row.record_type != "exchange_instrument"
|
||||
or row.amount is not None
|
||||
):
|
||||
raise SnapshotValidationError("invalid_exchange_adapter_record")
|
||||
organization, reason = _identity(row, index)
|
||||
if reason:
|
||||
counts[reason] += 1
|
||||
counts["skipped" if reason == "outside_directory" else "quarantined"] += 1
|
||||
if reason == "outside_directory":
|
||||
continue
|
||||
row.payload["provenance"] = {
|
||||
**row.payload.get("provenance", {}),
|
||||
"artifact_id": str(raw.uid),
|
||||
"source_published_at": raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
}
|
||||
row_number += 1
|
||||
pending.append(
|
||||
ParserStagedRecord(
|
||||
artifact=aggregate,
|
||||
row_number=row_number,
|
||||
organization=organization,
|
||||
external_id=row.external_id,
|
||||
record_type=row.record_type,
|
||||
raw_data=row.payload.get("upstream", {}),
|
||||
normalized_data={
|
||||
"title": row.title,
|
||||
"record_date": row.record_date,
|
||||
"amount": None,
|
||||
"status": row.status,
|
||||
"url": row.url,
|
||||
"payload": row.payload,
|
||||
},
|
||||
disposition="quarantined" if reason else "staged",
|
||||
reason=reason,
|
||||
)
|
||||
)
|
||||
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
|
||||
stage_records(pending)
|
||||
if on_progress:
|
||||
on_progress(
|
||||
sum(c["parsed"] for c in counters.values()),
|
||||
0,
|
||||
aggregate.staged_records.count(),
|
||||
sum(c["quarantined"] for c in counters.values()),
|
||||
)
|
||||
stage_records(pending)
|
||||
|
||||
|
||||
def _save_manifest(aggregate, raws, counters):
|
||||
aggregate.parsed_count = sum(c["parsed"] for c in counters.values())
|
||||
aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values())
|
||||
aggregate.metadata = {
|
||||
**aggregate.metadata,
|
||||
"inputs": [
|
||||
{
|
||||
"exchange": raw.metadata["dataset"],
|
||||
"artifact_id": str(raw.uid),
|
||||
"checksum_sha256": raw.sha256,
|
||||
"size_bytes": raw.size_bytes,
|
||||
"published_at": raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
"parsed_records_count": raw.parsed_count,
|
||||
}
|
||||
for raw in raws
|
||||
],
|
||||
"adapter_counters": {key: dict(value) for key, value in counters.items()},
|
||||
"skipped_records_count": sum(c["skipped"] for c in counters.values()),
|
||||
"reasons": dict(
|
||||
sum(
|
||||
(
|
||||
Counter(
|
||||
{
|
||||
k: v
|
||||
for k, v in c.items()
|
||||
if k not in {"parsed", "skipped", "quarantined"}
|
||||
}
|
||||
)
|
||||
for c in counters.values()
|
||||
),
|
||||
Counter(),
|
||||
)
|
||||
),
|
||||
}
|
||||
aggregate.rejection_reasons = aggregate.metadata["reasons"]
|
||||
dates = [raw.source_published_at for raw in raws if raw.source_published_at]
|
||||
aggregate.source_published_at = max(dates) if dates else None
|
||||
save_artifact_file(
|
||||
aggregate,
|
||||
BytesIO(
|
||||
json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def refresh_stock_exchange_activity(
|
||||
*,
|
||||
load_batch: int,
|
||||
uploaded_by_id: int | None = None,
|
||||
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
|
||||
on_progress: Callable[[int, int, int, int], None] | None = None,
|
||||
) -> tuple[ParserSourceArtifact, SnapshotResult]:
|
||||
aggregate = ParserSourceArtifact.objects.create(
|
||||
source=EXCHANGE_SOURCE,
|
||||
original_name="exchange-manifest.json",
|
||||
content_type="application/json",
|
||||
version="spb-csv-v1+moex-iss-v1",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
)
|
||||
raws = []
|
||||
counters = {"spb": Counter(), "moex": Counter()}
|
||||
pending = []
|
||||
try:
|
||||
index = OwnOrganizationIndex()
|
||||
spb = download_spb_artifact(
|
||||
load_batch=load_batch, uploaded_by_id=uploaded_by_id
|
||||
)
|
||||
raws.append(spb)
|
||||
spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)}
|
||||
spb.save(update_fields=("metadata",))
|
||||
with spb.file.open("rb") as handle:
|
||||
_stage(
|
||||
iter_spb_records(handle),
|
||||
exchange="spb",
|
||||
raw=spb,
|
||||
aggregate=aggregate,
|
||||
index=index,
|
||||
counts=counters["spb"],
|
||||
counters=counters,
|
||||
pending=pending,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
spb.parsed_count = counters["spb"]["parsed"]
|
||||
spb.status = "parsed"
|
||||
spb.save()
|
||||
moex = ParserSourceArtifact.objects.create(
|
||||
source=EXCHANGE_SOURCE,
|
||||
original_name="moex-responses.ndjson",
|
||||
content_type="application/x-ndjson",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
metadata={
|
||||
"dataset": "moex",
|
||||
"input_only": True,
|
||||
"aggregate_artifact_id": str(aggregate.uid),
|
||||
},
|
||||
)
|
||||
raws.append(moex)
|
||||
_stage(
|
||||
iter_moex_records(organization_index=index, artifact=moex),
|
||||
exchange="moex",
|
||||
raw=moex,
|
||||
aggregate=aggregate,
|
||||
index=index,
|
||||
counts=counters["moex"],
|
||||
counters=counters,
|
||||
pending=pending,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
extra = moex.metadata.get("diagnostics", {})
|
||||
counters["moex"]["parsed"] = moex.parsed_count
|
||||
counters["moex"]["skipped"] += extra.get("outside_directory", 0)
|
||||
counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0)
|
||||
counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get(
|
||||
"missing_or_invalid_issuer_inn", 0
|
||||
)
|
||||
# Raw checksum/date are final only once the generator reaches EOF.
|
||||
for staged in aggregate.staged_records.all().iterator(
|
||||
chunk_size=SNAPSHOT_CHUNK_SIZE
|
||||
):
|
||||
raw = (
|
||||
spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex
|
||||
)
|
||||
staged.normalized_data["payload"]["provenance"].update(
|
||||
checksum_sha256=raw.sha256,
|
||||
source_published_at=raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
)
|
||||
staged.save(update_fields=("normalized_data",))
|
||||
_save_manifest(aggregate, raws, counters)
|
||||
aggregate.status = "parsed"
|
||||
aggregate.save()
|
||||
return aggregate, publish_snapshot(aggregate, on_publish=on_publish)
|
||||
except Exception as exc:
|
||||
# If discovery failed before returning the SPB input, retain its bounded diagnostics too.
|
||||
if not raws:
|
||||
raws = list(
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source=EXCHANGE_SOURCE,
|
||||
load_batch=load_batch,
|
||||
metadata__input_only=True,
|
||||
)
|
||||
)
|
||||
_save_manifest(aggregate, raws, counters)
|
||||
for raw in raws:
|
||||
reject_snapshot(raw, exc)
|
||||
reject_snapshot(aggregate, exc)
|
||||
raise
|
||||
Reference in New Issue
Block a user