255 lines
9.1 KiB
Python
255 lines
9.1 KiB
Python
"""Two private exchange inputs, one validated atomic canonical snapshot."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from collections import Counter
|
|
from collections.abc import Callable
|
|
from io import BytesIO
|
|
|
|
from apps.parsers.exchange_moex import iter_moex_records
|
|
from apps.parsers.exchange_spb import (
|
|
EXCHANGE_SOURCE,
|
|
download_spb_artifact,
|
|
iter_spb_records,
|
|
)
|
|
from apps.parsers.fssp_csv import valid_legal_inn
|
|
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
|
from apps.parsers.registry_snapshots import (
|
|
SNAPSHOT_CHUNK_SIZE,
|
|
OwnOrganizationIndex,
|
|
SnapshotResult,
|
|
SnapshotValidationError,
|
|
publish_snapshot,
|
|
reject_snapshot,
|
|
save_artifact_file,
|
|
stage_records,
|
|
)
|
|
|
|
|
|
def _identity(row, index):
|
|
if not valid_legal_inn(row.inn):
|
|
return None, "invalid_issuer_inn"
|
|
organization, reason = index.resolve(inn=row.inn, ogrn="")
|
|
if not reason and (
|
|
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
|
|
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
|
|
):
|
|
return organization, "invalid_canonical_requisites"
|
|
return organization, reason
|
|
|
|
|
|
def _stage(
|
|
rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress
|
|
):
|
|
row_number = aggregate.staged_records.count()
|
|
for row in rows:
|
|
counts["parsed"] += 1
|
|
if (
|
|
row.payload.get("exchange") != exchange
|
|
or not row.external_id.startswith(exchange + ":")
|
|
or row.record_type != "exchange_instrument"
|
|
or row.amount is not None
|
|
):
|
|
raise SnapshotValidationError("invalid_exchange_adapter_record")
|
|
organization, reason = _identity(row, index)
|
|
if reason:
|
|
counts[reason] += 1
|
|
counts["skipped" if reason == "outside_directory" else "quarantined"] += 1
|
|
if reason == "outside_directory":
|
|
continue
|
|
row.payload["provenance"] = {
|
|
**row.payload.get("provenance", {}),
|
|
"artifact_id": str(raw.uid),
|
|
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
|
|
"source_published_at": raw.source_published_at.isoformat()
|
|
if raw.source_published_at
|
|
else None,
|
|
}
|
|
row_number += 1
|
|
pending.append(
|
|
ParserStagedRecord(
|
|
artifact=aggregate,
|
|
row_number=row_number,
|
|
organization=organization,
|
|
external_id=row.external_id,
|
|
record_type=row.record_type,
|
|
raw_data=row.payload.get("upstream", {}),
|
|
normalized_data={
|
|
"title": row.title,
|
|
"record_date": row.record_date,
|
|
"amount": None,
|
|
"status": row.status,
|
|
"url": row.url,
|
|
"payload": row.payload,
|
|
},
|
|
disposition="quarantined" if reason else "staged",
|
|
reason=reason,
|
|
)
|
|
)
|
|
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
|
|
stage_records(pending)
|
|
if on_progress:
|
|
on_progress(
|
|
sum(c["parsed"] for c in counters.values()),
|
|
0,
|
|
aggregate.staged_records.count(),
|
|
sum(c["quarantined"] for c in counters.values()),
|
|
)
|
|
stage_records(pending)
|
|
|
|
|
|
def _save_manifest(aggregate, raws, counters):
|
|
aggregate.parsed_count = sum(c["parsed"] for c in counters.values())
|
|
aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values())
|
|
aggregate.metadata = {
|
|
**aggregate.metadata,
|
|
"inputs": [
|
|
{
|
|
"exchange": raw.metadata["dataset"],
|
|
"artifact_id": str(raw.uid),
|
|
"fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"),
|
|
"checksum_sha256": raw.sha256,
|
|
"size_bytes": raw.size_bytes,
|
|
"published_at": raw.source_published_at.isoformat()
|
|
if raw.source_published_at
|
|
else None,
|
|
"parsed_records_count": raw.parsed_count,
|
|
}
|
|
for raw in raws
|
|
],
|
|
"adapter_counters": {key: dict(value) for key, value in counters.items()},
|
|
"skipped_records_count": sum(c["skipped"] for c in counters.values()),
|
|
"reasons": dict(
|
|
sum(
|
|
(
|
|
Counter(
|
|
{
|
|
k: v
|
|
for k, v in c.items()
|
|
if k not in {"parsed", "skipped", "quarantined"}
|
|
}
|
|
)
|
|
for c in counters.values()
|
|
),
|
|
Counter(),
|
|
)
|
|
),
|
|
}
|
|
aggregate.rejection_reasons = aggregate.metadata["reasons"]
|
|
dates = [raw.source_published_at for raw in raws if raw.source_published_at]
|
|
aggregate.source_published_at = max(dates) if dates else None
|
|
save_artifact_file(
|
|
aggregate,
|
|
BytesIO(
|
|
json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode()
|
|
),
|
|
)
|
|
|
|
|
|
def refresh_stock_exchange_activity(
|
|
*,
|
|
load_batch: int,
|
|
uploaded_by_id: int | None = None,
|
|
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
|
|
on_progress: Callable[[int, int, int, int], None] | None = None,
|
|
) -> tuple[ParserSourceArtifact, SnapshotResult]:
|
|
aggregate = ParserSourceArtifact.objects.create(
|
|
source=EXCHANGE_SOURCE,
|
|
original_name="exchange-manifest.json",
|
|
content_type="application/json",
|
|
version="spb-csv-v1+moex-iss-v1",
|
|
load_batch=load_batch,
|
|
uploaded_by_id=uploaded_by_id,
|
|
)
|
|
raws = []
|
|
counters = {"spb": Counter(), "moex": Counter()}
|
|
pending = []
|
|
try:
|
|
index = OwnOrganizationIndex()
|
|
spb = download_spb_artifact(
|
|
load_batch=load_batch, uploaded_by_id=uploaded_by_id
|
|
)
|
|
raws.append(spb)
|
|
spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)}
|
|
spb.save(update_fields=("metadata",))
|
|
with spb.file.open("rb") as handle:
|
|
_stage(
|
|
iter_spb_records(handle),
|
|
exchange="spb",
|
|
raw=spb,
|
|
aggregate=aggregate,
|
|
index=index,
|
|
counts=counters["spb"],
|
|
counters=counters,
|
|
pending=pending,
|
|
on_progress=on_progress,
|
|
)
|
|
spb.parsed_count = counters["spb"]["parsed"]
|
|
spb.status = "parsed"
|
|
spb.save()
|
|
moex = ParserSourceArtifact.objects.create(
|
|
source=EXCHANGE_SOURCE,
|
|
original_name="moex-responses.ndjson",
|
|
content_type="application/x-ndjson",
|
|
load_batch=load_batch,
|
|
uploaded_by_id=uploaded_by_id,
|
|
metadata={
|
|
"dataset": "moex",
|
|
"input_only": True,
|
|
"aggregate_artifact_id": str(aggregate.uid),
|
|
},
|
|
)
|
|
raws.append(moex)
|
|
_stage(
|
|
iter_moex_records(organization_index=index, artifact=moex),
|
|
exchange="moex",
|
|
raw=moex,
|
|
aggregate=aggregate,
|
|
index=index,
|
|
counts=counters["moex"],
|
|
counters=counters,
|
|
pending=pending,
|
|
on_progress=on_progress,
|
|
)
|
|
extra = moex.metadata.get("diagnostics", {})
|
|
counters["moex"]["parsed"] = moex.parsed_count
|
|
counters["moex"]["skipped"] += extra.get("outside_directory", 0)
|
|
counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0)
|
|
counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get(
|
|
"missing_or_invalid_issuer_inn", 0
|
|
)
|
|
# Raw checksum/date are final only once the generator reaches EOF.
|
|
for staged in aggregate.staged_records.all().iterator(
|
|
chunk_size=SNAPSHOT_CHUNK_SIZE
|
|
):
|
|
raw = (
|
|
spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex
|
|
)
|
|
staged.normalized_data["payload"]["provenance"].update(
|
|
checksum_sha256=raw.sha256,
|
|
source_published_at=raw.source_published_at.isoformat()
|
|
if raw.source_published_at
|
|
else None,
|
|
)
|
|
staged.save(update_fields=("normalized_data",))
|
|
_save_manifest(aggregate, raws, counters)
|
|
aggregate.status = "parsed"
|
|
aggregate.save()
|
|
return aggregate, publish_snapshot(aggregate, on_publish=on_publish)
|
|
except Exception as exc:
|
|
# If discovery failed before returning the SPB input, retain its bounded diagnostics too.
|
|
if not raws:
|
|
raws = list(
|
|
ParserSourceArtifact.objects.filter(
|
|
source=EXCHANGE_SOURCE,
|
|
load_batch=load_batch,
|
|
metadata__input_only=True,
|
|
)
|
|
)
|
|
_save_manifest(aggregate, raws, counters)
|
|
for raw in raws:
|
|
reject_snapshot(raw, exc)
|
|
reject_snapshot(aggregate, exc)
|
|
raise
|