feat: publish unified exchange activity snapshots

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:14:10 +02:00
parent e2cf384853
commit d69ba600d4
8 changed files with 775 additions and 0 deletions

View File

@@ -0,0 +1,252 @@
"""Two private exchange inputs, one validated atomic canonical snapshot."""
from __future__ import annotations
import json
import re
from collections import Counter
from collections.abc import Callable
from io import BytesIO
from apps.parsers.exchange_moex import iter_moex_records
from apps.parsers.exchange_spb import (
EXCHANGE_SOURCE,
download_spb_artifact,
iter_spb_records,
)
from apps.parsers.fssp_csv import valid_legal_inn
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from apps.parsers.registry_snapshots import (
SNAPSHOT_CHUNK_SIZE,
OwnOrganizationIndex,
SnapshotResult,
SnapshotValidationError,
publish_snapshot,
reject_snapshot,
save_artifact_file,
stage_records,
)
def _identity(row, index):
if not valid_legal_inn(row.inn):
return None, "invalid_issuer_inn"
organization, reason = index.resolve(inn=row.inn, ogrn="")
if not reason and (
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
):
return organization, "invalid_canonical_requisites"
return organization, reason
def _stage(
rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress
):
row_number = aggregate.staged_records.count()
for row in rows:
counts["parsed"] += 1
if (
row.payload.get("exchange") != exchange
or not row.external_id.startswith(exchange + ":")
or row.record_type != "exchange_instrument"
or row.amount is not None
):
raise SnapshotValidationError("invalid_exchange_adapter_record")
organization, reason = _identity(row, index)
if reason:
counts[reason] += 1
counts["skipped" if reason == "outside_directory" else "quarantined"] += 1
if reason == "outside_directory":
continue
row.payload["provenance"] = {
**row.payload.get("provenance", {}),
"artifact_id": str(raw.uid),
"source_published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
}
row_number += 1
pending.append(
ParserStagedRecord(
artifact=aggregate,
row_number=row_number,
organization=organization,
external_id=row.external_id,
record_type=row.record_type,
raw_data=row.payload.get("upstream", {}),
normalized_data={
"title": row.title,
"record_date": row.record_date,
"amount": None,
"status": row.status,
"url": row.url,
"payload": row.payload,
},
disposition="quarantined" if reason else "staged",
reason=reason,
)
)
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
stage_records(pending)
if on_progress:
on_progress(
sum(c["parsed"] for c in counters.values()),
0,
aggregate.staged_records.count(),
sum(c["quarantined"] for c in counters.values()),
)
stage_records(pending)
def _save_manifest(aggregate, raws, counters):
aggregate.parsed_count = sum(c["parsed"] for c in counters.values())
aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values())
aggregate.metadata = {
**aggregate.metadata,
"inputs": [
{
"exchange": raw.metadata["dataset"],
"artifact_id": str(raw.uid),
"checksum_sha256": raw.sha256,
"size_bytes": raw.size_bytes,
"published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
"parsed_records_count": raw.parsed_count,
}
for raw in raws
],
"adapter_counters": {key: dict(value) for key, value in counters.items()},
"skipped_records_count": sum(c["skipped"] for c in counters.values()),
"reasons": dict(
sum(
(
Counter(
{
k: v
for k, v in c.items()
if k not in {"parsed", "skipped", "quarantined"}
}
)
for c in counters.values()
),
Counter(),
)
),
}
aggregate.rejection_reasons = aggregate.metadata["reasons"]
dates = [raw.source_published_at for raw in raws if raw.source_published_at]
aggregate.source_published_at = max(dates) if dates else None
save_artifact_file(
aggregate,
BytesIO(
json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode()
),
)
def refresh_stock_exchange_activity(
*,
load_batch: int,
uploaded_by_id: int | None = None,
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
on_progress: Callable[[int, int, int, int], None] | None = None,
) -> tuple[ParserSourceArtifact, SnapshotResult]:
aggregate = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="exchange-manifest.json",
content_type="application/json",
version="spb-csv-v1+moex-iss-v1",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
)
raws = []
counters = {"spb": Counter(), "moex": Counter()}
pending = []
try:
index = OwnOrganizationIndex()
spb = download_spb_artifact(
load_batch=load_batch, uploaded_by_id=uploaded_by_id
)
raws.append(spb)
spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)}
spb.save(update_fields=("metadata",))
with spb.file.open("rb") as handle:
_stage(
iter_spb_records(handle),
exchange="spb",
raw=spb,
aggregate=aggregate,
index=index,
counts=counters["spb"],
counters=counters,
pending=pending,
on_progress=on_progress,
)
spb.parsed_count = counters["spb"]["parsed"]
spb.status = "parsed"
spb.save()
moex = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="moex-responses.ndjson",
content_type="application/x-ndjson",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
metadata={
"dataset": "moex",
"input_only": True,
"aggregate_artifact_id": str(aggregate.uid),
},
)
raws.append(moex)
_stage(
iter_moex_records(organization_index=index, artifact=moex),
exchange="moex",
raw=moex,
aggregate=aggregate,
index=index,
counts=counters["moex"],
counters=counters,
pending=pending,
on_progress=on_progress,
)
extra = moex.metadata.get("diagnostics", {})
counters["moex"]["parsed"] = moex.parsed_count
counters["moex"]["skipped"] += extra.get("outside_directory", 0)
counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0)
counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get(
"missing_or_invalid_issuer_inn", 0
)
# Raw checksum/date are final only once the generator reaches EOF.
for staged in aggregate.staged_records.all().iterator(
chunk_size=SNAPSHOT_CHUNK_SIZE
):
raw = (
spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex
)
staged.normalized_data["payload"]["provenance"].update(
checksum_sha256=raw.sha256,
source_published_at=raw.source_published_at.isoformat()
if raw.source_published_at
else None,
)
staged.save(update_fields=("normalized_data",))
_save_manifest(aggregate, raws, counters)
aggregate.status = "parsed"
aggregate.save()
return aggregate, publish_snapshot(aggregate, on_publish=on_publish)
except Exception as exc:
# If discovery failed before returning the SPB input, retain its bounded diagnostics too.
if not raws:
raws = list(
ParserSourceArtifact.objects.filter(
source=EXCHANGE_SOURCE,
load_batch=load_batch,
metadata__input_only=True,
)
)
_save_manifest(aggregate, raws, counters)
for raw in raws:
reject_snapshot(raw, exc)
reject_snapshot(aggregate, exc)
raise

View File

@@ -0,0 +1,220 @@
# Generated by Django 3.2.25 on 2026-10-05 15:12
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("parsers", "0045_checko_disabled_schedules"),
]
operations = [
migrations.AlterField(
model_name="genericparserrecord",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
("hh", "Вакансии HeadHunter"),
("superjob", "Вакансии SuperJob"),
],
db_index=True,
help_text="Источник данных",
max_length=50,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parserbatchsequence",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
],
help_text="Источник данных",
max_length=50,
unique=True,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parserloadlog",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
],
db_index=True,
help_text="Источник данных",
max_length=50,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parsersourceartifact",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
],
db_index=True,
max_length=50,
),
),
]

View File

@@ -66,6 +66,8 @@ class ParserLoadLog(TimestampMixin, models.Model):
_("Исполнительные производства ФССП"),
)
STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность")
CHECKO_LEASING = "checko_leasing", _("Лизинг")
CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций")
CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций")

View File

@@ -0,0 +1,74 @@
# Generated by Django 3.2.25 on 2026-10-05 15:12
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("organizations", "0017_checko_sections"),
]
operations = [
migrations.CreateModel(
name="StockExchangeActivityExtension",
fields=[
(
"organizationsourceextension_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="organizations.organizationsourceextension",
),
),
],
options={
"verbose_name": "биржевая активность",
"verbose_name_plural": "биржевая активность",
"db_table": "organizations_stock_exchange_activity_extension",
},
bases=("organizations.organizationsourceextension",),
),
migrations.AlterField(
model_name="organizationsourceextension",
name="source_group",
field=models.CharField(
choices=[
("financial_indicators", "Финансово-экономические показатели"),
("government_procurements", "Государственные закупки"),
("industrial_production", "Производители и продукция России"),
("planned_inspections", "Плановые проверки"),
("bankruptcy", "Сведения о процедурах банкротства"),
("defense_suppliers", "Недобросовестные поставщики ГОЗ"),
("arbitration", "Арбитражные дела"),
("security_registries", "Реестры по информационной безопасности"),
("vacancies", "Вакансии"),
("electronic_document_exchange", "Электронный документооборот"),
("media_mentions", "Упоминания в СМИ"),
("sanctions", "Санкции"),
("government_support", "Государственная поддержка"),
("sro_membership", "Членство в СРО"),
(
"budget_process_registry",
"Реестр участников бюджетного процесса",
),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
("enforcement_proceedings", "Исполнительные производства"),
("stock_exchange_activity", "Биржевая активность"),
("leasing", "Лизинг"),
("company_messages", "Сообщения организаций"),
("company_licenses", "Лицензии организаций"),
],
db_index=True,
max_length=64,
verbose_name="группа источников",
),
),
]

View File

@@ -57,6 +57,8 @@ class SourceGroup(models.TextChoices):
_("Исполнительные производства"),
)
STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность")
LEASING = "leasing", _("Лизинг")
COMPANY_MESSAGES = "company_messages", _("Сообщения организаций")
COMPANY_LICENSES = "company_licenses", _("Лицензии организаций")
@@ -1070,3 +1072,14 @@ class CompanyLicenseExtension(OrganizationSourceExtension):
db_table = "organizations_company_license_extension"
verbose_name = _("лицензии организации")
verbose_name_plural = _("лицензии организации")
class StockExchangeActivityExtension(OrganizationSourceExtension):
"""Combined SPB and MOEX observations for one canonical organization."""
source_group_value = SourceGroup.STOCK_EXCHANGE_ACTIVITY
class Meta:
db_table = "organizations_stock_exchange_activity_extension"
verbose_name = _("биржевая активность")
verbose_name_plural = _("биржевая активность")

View File

@@ -29,6 +29,7 @@ from organizations.models import (
SecurityRegistryExtension,
SourceGroup,
SroMembershipExtension,
StockExchangeActivityExtension,
VacancyExtension,
)
@@ -45,6 +46,13 @@ class SourceGroupDescriptor:
SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = {
ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY: SourceGroupDescriptor(
source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
source_group=SourceGroup.STOCK_EXCHANGE_ACTIVITY,
record_type="exchange_instrument",
title="Биржевая активность",
extension_model=StockExchangeActivityExtension,
),
ParserLoadLog.Source.CHECKO_LICENSES: SourceGroupDescriptor(
source=ParserLoadLog.Source.CHECKO_LICENSES,
source_group=SourceGroup.COMPANY_LICENSES,

View File

@@ -528,3 +528,11 @@ SRO_REQUEST_INTERVAL_SECONDS = float(os.getenv("SRO_REQUEST_INTERVAL_SECONDS", "
SRO_HTTP_MAX_RESPONSE_BYTES = int(os.getenv("SRO_HTTP_MAX_RESPONSE_BYTES", "2097152"))
SRO_HTTP_TIMEOUT_SECONDS = int(os.getenv("SRO_HTTP_TIMEOUT_SECONDS", "30"))
SRO_MAX_REQUESTS_PER_RUN = int(os.getenv("SRO_MAX_REQUESTS_PER_RUN", "10000"))
# Public exchange collection bounds; TLS verification stays enabled.
EXCHANGE_SPB_MAX_BYTES = 50 * 1024**2
EXCHANGE_SPB_MAX_ROWS = 100000
EXCHANGE_MOEX_MAX_RESPONSE_BYTES = 10 * 1024**2
EXCHANGE_MOEX_MAX_PAGES = 1000
EXCHANGE_MOEX_MAX_REQUESTS = 10000
EXCHANGE_MOEX_MAX_DURATION_ROWS = 100000