"""Two private exchange inputs, one validated atomic canonical snapshot.""" from __future__ import annotations import json import re from collections import Counter from collections.abc import Callable from io import BytesIO from apps.parsers.exchange_moex import iter_moex_records from apps.parsers.exchange_spb import ( EXCHANGE_SOURCE, download_spb_artifact, iter_spb_records, ) from apps.parsers.fssp_csv import valid_legal_inn from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord from apps.parsers.registry_snapshots import ( SNAPSHOT_CHUNK_SIZE, OwnOrganizationIndex, SnapshotResult, SnapshotValidationError, publish_snapshot, reject_snapshot, save_artifact_file, stage_records, ) def _identity(row, index): if not valid_legal_inn(row.inn): return None, "invalid_issuer_inn" organization, reason = index.resolve(inn=row.inn, ogrn="") if not reason and ( not re.fullmatch(r"[0-9]{13}", organization.ogrn or "") or not re.fullmatch(r"[0-9]{8}", organization.okpo or "") ): return organization, "invalid_canonical_requisites" return organization, reason def _stage( rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress ): row_number = aggregate.staged_records.count() for row in rows: counts["parsed"] += 1 if ( row.payload.get("exchange") != exchange or not row.external_id.startswith(exchange + ":") or row.record_type != "exchange_instrument" or row.amount is not None ): raise SnapshotValidationError("invalid_exchange_adapter_record") organization, reason = _identity(row, index) if reason: counts[reason] += 1 counts["skipped" if reason == "outside_directory" else "quarantined"] += 1 if reason == "outside_directory": continue row.payload["provenance"] = { **row.payload.get("provenance", {}), "artifact_id": str(raw.uid), "fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"), "source_published_at": raw.source_published_at.isoformat() if raw.source_published_at else None, } row_number += 1 pending.append( ParserStagedRecord( artifact=aggregate, row_number=row_number, organization=organization, external_id=row.external_id, record_type=row.record_type, raw_data=row.payload.get("upstream", {}), normalized_data={ "title": row.title, "record_date": row.record_date, "amount": None, "status": row.status, "url": row.url, "payload": row.payload, }, disposition="quarantined" if reason else "staged", reason=reason, ) ) if len(pending) >= SNAPSHOT_CHUNK_SIZE: stage_records(pending) if on_progress: on_progress( sum(c["parsed"] for c in counters.values()), 0, aggregate.staged_records.count(), sum(c["quarantined"] for c in counters.values()), ) stage_records(pending) def _save_manifest(aggregate, raws, counters): aggregate.parsed_count = sum(c["parsed"] for c in counters.values()) aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values()) aggregate.metadata = { **aggregate.metadata, "inputs": [ { "exchange": raw.metadata["dataset"], "artifact_id": str(raw.uid), "fetched_at": raw.created_at.isoformat().replace("+00:00", "Z"), "checksum_sha256": raw.sha256, "size_bytes": raw.size_bytes, "published_at": raw.source_published_at.isoformat() if raw.source_published_at else None, "parsed_records_count": raw.parsed_count, } for raw in raws ], "adapter_counters": {key: dict(value) for key, value in counters.items()}, "skipped_records_count": sum(c["skipped"] for c in counters.values()), "reasons": dict( sum( ( Counter( { k: v for k, v in c.items() if k not in {"parsed", "skipped", "quarantined"} } ) for c in counters.values() ), Counter(), ) ), } aggregate.rejection_reasons = aggregate.metadata["reasons"] dates = [raw.source_published_at for raw in raws if raw.source_published_at] aggregate.source_published_at = max(dates) if dates else None save_artifact_file( aggregate, BytesIO( json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode() ), ) def refresh_stock_exchange_activity( *, load_batch: int, uploaded_by_id: int | None = None, on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None, on_progress: Callable[[int, int, int, int], None] | None = None, ) -> tuple[ParserSourceArtifact, SnapshotResult]: aggregate = ParserSourceArtifact.objects.create( source=EXCHANGE_SOURCE, original_name="exchange-manifest.json", content_type="application/json", version="spb-csv-v1+moex-iss-v1", load_batch=load_batch, uploaded_by_id=uploaded_by_id, ) raws = [] counters = {"spb": Counter(), "moex": Counter()} pending = [] try: index = OwnOrganizationIndex() spb = download_spb_artifact( load_batch=load_batch, uploaded_by_id=uploaded_by_id ) raws.append(spb) spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)} spb.save(update_fields=("metadata",)) with spb.file.open("rb") as handle: _stage( iter_spb_records(handle), exchange="spb", raw=spb, aggregate=aggregate, index=index, counts=counters["spb"], counters=counters, pending=pending, on_progress=on_progress, ) spb.parsed_count = counters["spb"]["parsed"] spb.status = "parsed" spb.save() moex = ParserSourceArtifact.objects.create( source=EXCHANGE_SOURCE, original_name="moex-responses.ndjson", content_type="application/x-ndjson", load_batch=load_batch, uploaded_by_id=uploaded_by_id, metadata={ "dataset": "moex", "input_only": True, "aggregate_artifact_id": str(aggregate.uid), }, ) raws.append(moex) _stage( iter_moex_records(organization_index=index, artifact=moex), exchange="moex", raw=moex, aggregate=aggregate, index=index, counts=counters["moex"], counters=counters, pending=pending, on_progress=on_progress, ) extra = moex.metadata.get("diagnostics", {}) counters["moex"]["parsed"] = moex.parsed_count counters["moex"]["skipped"] += extra.get("outside_directory", 0) counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0) counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get( "missing_or_invalid_issuer_inn", 0 ) # Raw checksum/date are final only once the generator reaches EOF. for staged in aggregate.staged_records.all().iterator( chunk_size=SNAPSHOT_CHUNK_SIZE ): raw = ( spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex ) staged.normalized_data["payload"]["provenance"].update( checksum_sha256=raw.sha256, source_published_at=raw.source_published_at.isoformat() if raw.source_published_at else None, ) staged.save(update_fields=("normalized_data",)) _save_manifest(aggregate, raws, counters) aggregate.status = "parsed" aggregate.save() return aggregate, publish_snapshot(aggregate, on_publish=on_publish) except Exception as exc: # If discovery failed before returning the SPB input, retain its bounded diagnostics too. if not raws: raws = list( ParserSourceArtifact.objects.filter( source=EXCHANGE_SOURCE, load_batch=load_batch, metadata__input_only=True, ) ) _save_manifest(aggregate, raws, counters) for raw in raws: reject_snapshot(raw, exc) reject_snapshot(aggregate, exc) raise