feat: publish unified exchange activity snapshots
This commit is contained in:
252
src/apps/parsers/exchange_snapshot.py
Normal file
252
src/apps/parsers/exchange_snapshot.py
Normal file
@@ -0,0 +1,252 @@
|
||||
"""Two private exchange inputs, one validated atomic canonical snapshot."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections import Counter
|
||||
from collections.abc import Callable
|
||||
from io import BytesIO
|
||||
|
||||
from apps.parsers.exchange_moex import iter_moex_records
|
||||
from apps.parsers.exchange_spb import (
|
||||
EXCHANGE_SOURCE,
|
||||
download_spb_artifact,
|
||||
iter_spb_records,
|
||||
)
|
||||
from apps.parsers.fssp_csv import valid_legal_inn
|
||||
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
|
||||
from apps.parsers.registry_snapshots import (
|
||||
SNAPSHOT_CHUNK_SIZE,
|
||||
OwnOrganizationIndex,
|
||||
SnapshotResult,
|
||||
SnapshotValidationError,
|
||||
publish_snapshot,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
stage_records,
|
||||
)
|
||||
|
||||
|
||||
def _identity(row, index):
|
||||
if not valid_legal_inn(row.inn):
|
||||
return None, "invalid_issuer_inn"
|
||||
organization, reason = index.resolve(inn=row.inn, ogrn="")
|
||||
if not reason and (
|
||||
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
|
||||
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
|
||||
):
|
||||
return organization, "invalid_canonical_requisites"
|
||||
return organization, reason
|
||||
|
||||
|
||||
def _stage(
|
||||
rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress
|
||||
):
|
||||
row_number = aggregate.staged_records.count()
|
||||
for row in rows:
|
||||
counts["parsed"] += 1
|
||||
if (
|
||||
row.payload.get("exchange") != exchange
|
||||
or not row.external_id.startswith(exchange + ":")
|
||||
or row.record_type != "exchange_instrument"
|
||||
or row.amount is not None
|
||||
):
|
||||
raise SnapshotValidationError("invalid_exchange_adapter_record")
|
||||
organization, reason = _identity(row, index)
|
||||
if reason:
|
||||
counts[reason] += 1
|
||||
counts["skipped" if reason == "outside_directory" else "quarantined"] += 1
|
||||
if reason == "outside_directory":
|
||||
continue
|
||||
row.payload["provenance"] = {
|
||||
**row.payload.get("provenance", {}),
|
||||
"artifact_id": str(raw.uid),
|
||||
"source_published_at": raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
}
|
||||
row_number += 1
|
||||
pending.append(
|
||||
ParserStagedRecord(
|
||||
artifact=aggregate,
|
||||
row_number=row_number,
|
||||
organization=organization,
|
||||
external_id=row.external_id,
|
||||
record_type=row.record_type,
|
||||
raw_data=row.payload.get("upstream", {}),
|
||||
normalized_data={
|
||||
"title": row.title,
|
||||
"record_date": row.record_date,
|
||||
"amount": None,
|
||||
"status": row.status,
|
||||
"url": row.url,
|
||||
"payload": row.payload,
|
||||
},
|
||||
disposition="quarantined" if reason else "staged",
|
||||
reason=reason,
|
||||
)
|
||||
)
|
||||
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
|
||||
stage_records(pending)
|
||||
if on_progress:
|
||||
on_progress(
|
||||
sum(c["parsed"] for c in counters.values()),
|
||||
0,
|
||||
aggregate.staged_records.count(),
|
||||
sum(c["quarantined"] for c in counters.values()),
|
||||
)
|
||||
stage_records(pending)
|
||||
|
||||
|
||||
def _save_manifest(aggregate, raws, counters):
|
||||
aggregate.parsed_count = sum(c["parsed"] for c in counters.values())
|
||||
aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values())
|
||||
aggregate.metadata = {
|
||||
**aggregate.metadata,
|
||||
"inputs": [
|
||||
{
|
||||
"exchange": raw.metadata["dataset"],
|
||||
"artifact_id": str(raw.uid),
|
||||
"checksum_sha256": raw.sha256,
|
||||
"size_bytes": raw.size_bytes,
|
||||
"published_at": raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
"parsed_records_count": raw.parsed_count,
|
||||
}
|
||||
for raw in raws
|
||||
],
|
||||
"adapter_counters": {key: dict(value) for key, value in counters.items()},
|
||||
"skipped_records_count": sum(c["skipped"] for c in counters.values()),
|
||||
"reasons": dict(
|
||||
sum(
|
||||
(
|
||||
Counter(
|
||||
{
|
||||
k: v
|
||||
for k, v in c.items()
|
||||
if k not in {"parsed", "skipped", "quarantined"}
|
||||
}
|
||||
)
|
||||
for c in counters.values()
|
||||
),
|
||||
Counter(),
|
||||
)
|
||||
),
|
||||
}
|
||||
aggregate.rejection_reasons = aggregate.metadata["reasons"]
|
||||
dates = [raw.source_published_at for raw in raws if raw.source_published_at]
|
||||
aggregate.source_published_at = max(dates) if dates else None
|
||||
save_artifact_file(
|
||||
aggregate,
|
||||
BytesIO(
|
||||
json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode()
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def refresh_stock_exchange_activity(
|
||||
*,
|
||||
load_batch: int,
|
||||
uploaded_by_id: int | None = None,
|
||||
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
|
||||
on_progress: Callable[[int, int, int, int], None] | None = None,
|
||||
) -> tuple[ParserSourceArtifact, SnapshotResult]:
|
||||
aggregate = ParserSourceArtifact.objects.create(
|
||||
source=EXCHANGE_SOURCE,
|
||||
original_name="exchange-manifest.json",
|
||||
content_type="application/json",
|
||||
version="spb-csv-v1+moex-iss-v1",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
)
|
||||
raws = []
|
||||
counters = {"spb": Counter(), "moex": Counter()}
|
||||
pending = []
|
||||
try:
|
||||
index = OwnOrganizationIndex()
|
||||
spb = download_spb_artifact(
|
||||
load_batch=load_batch, uploaded_by_id=uploaded_by_id
|
||||
)
|
||||
raws.append(spb)
|
||||
spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)}
|
||||
spb.save(update_fields=("metadata",))
|
||||
with spb.file.open("rb") as handle:
|
||||
_stage(
|
||||
iter_spb_records(handle),
|
||||
exchange="spb",
|
||||
raw=spb,
|
||||
aggregate=aggregate,
|
||||
index=index,
|
||||
counts=counters["spb"],
|
||||
counters=counters,
|
||||
pending=pending,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
spb.parsed_count = counters["spb"]["parsed"]
|
||||
spb.status = "parsed"
|
||||
spb.save()
|
||||
moex = ParserSourceArtifact.objects.create(
|
||||
source=EXCHANGE_SOURCE,
|
||||
original_name="moex-responses.ndjson",
|
||||
content_type="application/x-ndjson",
|
||||
load_batch=load_batch,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
metadata={
|
||||
"dataset": "moex",
|
||||
"input_only": True,
|
||||
"aggregate_artifact_id": str(aggregate.uid),
|
||||
},
|
||||
)
|
||||
raws.append(moex)
|
||||
_stage(
|
||||
iter_moex_records(organization_index=index, artifact=moex),
|
||||
exchange="moex",
|
||||
raw=moex,
|
||||
aggregate=aggregate,
|
||||
index=index,
|
||||
counts=counters["moex"],
|
||||
counters=counters,
|
||||
pending=pending,
|
||||
on_progress=on_progress,
|
||||
)
|
||||
extra = moex.metadata.get("diagnostics", {})
|
||||
counters["moex"]["parsed"] = moex.parsed_count
|
||||
counters["moex"]["skipped"] += extra.get("outside_directory", 0)
|
||||
counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0)
|
||||
counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get(
|
||||
"missing_or_invalid_issuer_inn", 0
|
||||
)
|
||||
# Raw checksum/date are final only once the generator reaches EOF.
|
||||
for staged in aggregate.staged_records.all().iterator(
|
||||
chunk_size=SNAPSHOT_CHUNK_SIZE
|
||||
):
|
||||
raw = (
|
||||
spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex
|
||||
)
|
||||
staged.normalized_data["payload"]["provenance"].update(
|
||||
checksum_sha256=raw.sha256,
|
||||
source_published_at=raw.source_published_at.isoformat()
|
||||
if raw.source_published_at
|
||||
else None,
|
||||
)
|
||||
staged.save(update_fields=("normalized_data",))
|
||||
_save_manifest(aggregate, raws, counters)
|
||||
aggregate.status = "parsed"
|
||||
aggregate.save()
|
||||
return aggregate, publish_snapshot(aggregate, on_publish=on_publish)
|
||||
except Exception as exc:
|
||||
# If discovery failed before returning the SPB input, retain its bounded diagnostics too.
|
||||
if not raws:
|
||||
raws = list(
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source=EXCHANGE_SOURCE,
|
||||
load_batch=load_batch,
|
||||
metadata__input_only=True,
|
||||
)
|
||||
)
|
||||
_save_manifest(aggregate, raws, counters)
|
||||
for raw in raws:
|
||||
reject_snapshot(raw, exc)
|
||||
reject_snapshot(aggregate, exc)
|
||||
raise
|
||||
220
src/apps/parsers/migrations/0046_stock_exchange_activity.py
Normal file
220
src/apps/parsers/migrations/0046_stock_exchange_activity.py
Normal file
@@ -0,0 +1,220 @@
|
||||
# Generated by Django 3.2.25 on 2026-10-05 15:12
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("parsers", "0045_checko_disabled_schedules"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="genericparserrecord",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
("stock_exchange_activity", "Биржевая активность"),
|
||||
("checko_leasing", "Лизинг"),
|
||||
("checko_messages", "Сообщения организаций"),
|
||||
("checko_licenses", "Лицензии организаций"),
|
||||
("hh", "Вакансии HeadHunter"),
|
||||
("superjob", "Вакансии SuperJob"),
|
||||
],
|
||||
db_index=True,
|
||||
help_text="Источник данных",
|
||||
max_length=50,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="parserbatchsequence",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
("stock_exchange_activity", "Биржевая активность"),
|
||||
("checko_leasing", "Лизинг"),
|
||||
("checko_messages", "Сообщения организаций"),
|
||||
("checko_licenses", "Лицензии организаций"),
|
||||
],
|
||||
help_text="Источник данных",
|
||||
max_length=50,
|
||||
unique=True,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="parserloadlog",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
("stock_exchange_activity", "Биржевая активность"),
|
||||
("checko_leasing", "Лизинг"),
|
||||
("checko_messages", "Сообщения организаций"),
|
||||
("checko_licenses", "Лицензии организаций"),
|
||||
],
|
||||
db_index=True,
|
||||
help_text="Источник данных",
|
||||
max_length=50,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="parsersourceartifact",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("industrial", "Сертификаты промышленного производства"),
|
||||
("industrial_products", "Реестр промышленной продукции"),
|
||||
("manufactures", "Реестр производителей"),
|
||||
("inspections", "Единый реестр проверок"),
|
||||
("procurements", "Единая информационная система закупок"),
|
||||
("fns_reports", "Бухгалтерская отчетность ФНС"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("contracts", "Контракты ЕИС"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
("fas_goz", "Уклонение от ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("fedresurs_bankruptcy", "Банкротства Федресурс"),
|
||||
("fstec", "Реестры ФСТЭК"),
|
||||
("trudvsem", "Вакансии Работа России"),
|
||||
(
|
||||
"gosedo_address_directory",
|
||||
"Глобальный адресный справочник ГосЭДО",
|
||||
),
|
||||
("media_news", "Новости СМИ"),
|
||||
("ropk_sanctions", "РОПК — Санкции"),
|
||||
(
|
||||
"fns_sme_support_recipients",
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
),
|
||||
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
|
||||
("sro_membership_check", "Проверка членства в СРО"),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
(
|
||||
"fssp_enforcement_proceedings",
|
||||
"Исполнительные производства ФССП",
|
||||
),
|
||||
("stock_exchange_activity", "Биржевая активность"),
|
||||
("checko_leasing", "Лизинг"),
|
||||
("checko_messages", "Сообщения организаций"),
|
||||
("checko_licenses", "Лицензии организаций"),
|
||||
],
|
||||
db_index=True,
|
||||
max_length=50,
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -66,6 +66,8 @@ class ParserLoadLog(TimestampMixin, models.Model):
|
||||
_("Исполнительные производства ФССП"),
|
||||
)
|
||||
|
||||
STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность")
|
||||
|
||||
CHECKO_LEASING = "checko_leasing", _("Лизинг")
|
||||
CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций")
|
||||
CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций")
|
||||
|
||||
74
src/organizations/migrations/0018_stock_exchange_activity.py
Normal file
74
src/organizations/migrations/0018_stock_exchange_activity.py
Normal file
@@ -0,0 +1,74 @@
|
||||
# Generated by Django 3.2.25 on 2026-10-05 15:12
|
||||
|
||||
import django.db.models.deletion
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("organizations", "0017_checko_sections"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name="StockExchangeActivityExtension",
|
||||
fields=[
|
||||
(
|
||||
"organizationsourceextension_ptr",
|
||||
models.OneToOneField(
|
||||
auto_created=True,
|
||||
on_delete=django.db.models.deletion.CASCADE,
|
||||
parent_link=True,
|
||||
primary_key=True,
|
||||
serialize=False,
|
||||
to="organizations.organizationsourceextension",
|
||||
),
|
||||
),
|
||||
],
|
||||
options={
|
||||
"verbose_name": "биржевая активность",
|
||||
"verbose_name_plural": "биржевая активность",
|
||||
"db_table": "organizations_stock_exchange_activity_extension",
|
||||
},
|
||||
bases=("organizations.organizationsourceextension",),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="organizationsourceextension",
|
||||
name="source_group",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("financial_indicators", "Финансово-экономические показатели"),
|
||||
("government_procurements", "Государственные закупки"),
|
||||
("industrial_production", "Производители и продукция России"),
|
||||
("planned_inspections", "Плановые проверки"),
|
||||
("bankruptcy", "Сведения о процедурах банкротства"),
|
||||
("defense_suppliers", "Недобросовестные поставщики ГОЗ"),
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("security_registries", "Реестры по информационной безопасности"),
|
||||
("vacancies", "Вакансии"),
|
||||
("electronic_document_exchange", "Электронный документооборот"),
|
||||
("media_mentions", "Упоминания в СМИ"),
|
||||
("sanctions", "Санкции"),
|
||||
("government_support", "Государственная поддержка"),
|
||||
("sro_membership", "Членство в СРО"),
|
||||
(
|
||||
"budget_process_registry",
|
||||
"Реестр участников бюджетного процесса",
|
||||
),
|
||||
(
|
||||
"integrated_structure_documents",
|
||||
"Документы интегрированных структур",
|
||||
),
|
||||
("fedresurs_company_profile", "Карточки компаний Федресурса"),
|
||||
("enforcement_proceedings", "Исполнительные производства"),
|
||||
("stock_exchange_activity", "Биржевая активность"),
|
||||
("leasing", "Лизинг"),
|
||||
("company_messages", "Сообщения организаций"),
|
||||
("company_licenses", "Лицензии организаций"),
|
||||
],
|
||||
db_index=True,
|
||||
max_length=64,
|
||||
verbose_name="группа источников",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -57,6 +57,8 @@ class SourceGroup(models.TextChoices):
|
||||
_("Исполнительные производства"),
|
||||
)
|
||||
|
||||
STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность")
|
||||
|
||||
LEASING = "leasing", _("Лизинг")
|
||||
COMPANY_MESSAGES = "company_messages", _("Сообщения организаций")
|
||||
COMPANY_LICENSES = "company_licenses", _("Лицензии организаций")
|
||||
@@ -1070,3 +1072,14 @@ class CompanyLicenseExtension(OrganizationSourceExtension):
|
||||
db_table = "organizations_company_license_extension"
|
||||
verbose_name = _("лицензии организации")
|
||||
verbose_name_plural = _("лицензии организации")
|
||||
|
||||
|
||||
class StockExchangeActivityExtension(OrganizationSourceExtension):
|
||||
"""Combined SPB and MOEX observations for one canonical organization."""
|
||||
|
||||
source_group_value = SourceGroup.STOCK_EXCHANGE_ACTIVITY
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_stock_exchange_activity_extension"
|
||||
verbose_name = _("биржевая активность")
|
||||
verbose_name_plural = _("биржевая активность")
|
||||
|
||||
@@ -29,6 +29,7 @@ from organizations.models import (
|
||||
SecurityRegistryExtension,
|
||||
SourceGroup,
|
||||
SroMembershipExtension,
|
||||
StockExchangeActivityExtension,
|
||||
VacancyExtension,
|
||||
)
|
||||
|
||||
@@ -45,6 +46,13 @@ class SourceGroupDescriptor:
|
||||
|
||||
|
||||
SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = {
|
||||
ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY: SourceGroupDescriptor(
|
||||
source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
|
||||
source_group=SourceGroup.STOCK_EXCHANGE_ACTIVITY,
|
||||
record_type="exchange_instrument",
|
||||
title="Биржевая активность",
|
||||
extension_model=StockExchangeActivityExtension,
|
||||
),
|
||||
ParserLoadLog.Source.CHECKO_LICENSES: SourceGroupDescriptor(
|
||||
source=ParserLoadLog.Source.CHECKO_LICENSES,
|
||||
source_group=SourceGroup.COMPANY_LICENSES,
|
||||
|
||||
@@ -528,3 +528,11 @@ SRO_REQUEST_INTERVAL_SECONDS = float(os.getenv("SRO_REQUEST_INTERVAL_SECONDS", "
|
||||
SRO_HTTP_MAX_RESPONSE_BYTES = int(os.getenv("SRO_HTTP_MAX_RESPONSE_BYTES", "2097152"))
|
||||
SRO_HTTP_TIMEOUT_SECONDS = int(os.getenv("SRO_HTTP_TIMEOUT_SECONDS", "30"))
|
||||
SRO_MAX_REQUESTS_PER_RUN = int(os.getenv("SRO_MAX_REQUESTS_PER_RUN", "10000"))
|
||||
|
||||
# Public exchange collection bounds; TLS verification stays enabled.
|
||||
EXCHANGE_SPB_MAX_BYTES = 50 * 1024**2
|
||||
EXCHANGE_SPB_MAX_ROWS = 100000
|
||||
EXCHANGE_MOEX_MAX_RESPONSE_BYTES = 10 * 1024**2
|
||||
EXCHANGE_MOEX_MAX_PAGES = 1000
|
||||
EXCHANGE_MOEX_MAX_REQUESTS = 10000
|
||||
EXCHANGE_MOEX_MAX_DURATION_ROWS = 100000
|
||||
|
||||
198
tests/apps/parsers/test_exchange_snapshot.py
Normal file
198
tests/apps/parsers/test_exchange_snapshot.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""One source, two validated inputs, atomic publication with stable identities."""
|
||||
import csv
|
||||
import importlib
|
||||
from io import BytesIO, StringIO
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError, save_artifact_file
|
||||
from django.test import TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from organizations.source_ingestion import SourceRecordInput
|
||||
|
||||
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
|
||||
|
||||
|
||||
def adapter():
|
||||
return importlib.import_module("apps.parsers.exchange_snapshot")
|
||||
|
||||
|
||||
class ExchangeSnapshotTests(TestCase):
|
||||
def setUp(self):
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
self.tmp = TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.settings = override_settings(
|
||||
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
|
||||
MEDIA_ROOT=self.tmp.name + "/public",
|
||||
)
|
||||
self.settings.enable()
|
||||
self.addCleanup(self.settings.disable)
|
||||
self.org = Organization.objects.create(
|
||||
name="Canonical",
|
||||
inn="7707083893",
|
||||
ogrn="1027700132195",
|
||||
okpo="00032537",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
|
||||
def run_import(
|
||||
self, batch=1, fail=False, inn=None, duplicates=False, count=1, on_publish=None
|
||||
):
|
||||
module = adapter()
|
||||
|
||||
def download(**kwargs):
|
||||
raw = ParserSourceArtifact.objects.create(
|
||||
source="stock_exchange_activity",
|
||||
original_name="spb.csv",
|
||||
load_batch=batch,
|
||||
source_published_at="2026-10-05",
|
||||
metadata={"dataset": "spb", "input_only": True},
|
||||
)
|
||||
rows = list(
|
||||
csv.reader(
|
||||
StringIO(
|
||||
(FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")
|
||||
),
|
||||
delimiter=";",
|
||||
)
|
||||
)
|
||||
rows[1][rows[0].index("e_INN_code")] = (
|
||||
inn if inn is not None else self.org.inn
|
||||
)
|
||||
out = StringIO()
|
||||
csv.writer(out, delimiter=";").writerows(rows)
|
||||
save_artifact_file(raw, BytesIO(out.getvalue().encode()))
|
||||
raw.refresh_from_db()
|
||||
return raw
|
||||
|
||||
def moex(*, organization_index, artifact):
|
||||
save_artifact_file(artifact, BytesIO(b'{"synthetic":true}\n'))
|
||||
for n in range(count):
|
||||
if fail:
|
||||
raise SnapshotValidationError("synthetic_moex_failed")
|
||||
yield SourceRecordInput(
|
||||
external_id=f"moex:{0 if duplicates else n}",
|
||||
record_type="exchange_instrument",
|
||||
title="Synthetic bond",
|
||||
organization_name=self.org.name,
|
||||
inn=self.org.inn,
|
||||
payload={
|
||||
"exchange": "moex",
|
||||
"instrument_id": str(n),
|
||||
"issuer_inn": self.org.inn,
|
||||
"isin": "RU0000000001",
|
||||
"provenance": {},
|
||||
"upstream": {"id": n},
|
||||
},
|
||||
)
|
||||
artifact.parsed_count = count
|
||||
artifact.source_published_at = timezone.now().date()
|
||||
artifact.status = "parsed"
|
||||
artifact.save()
|
||||
|
||||
with patch.object(module, "download_spb_artifact", download), patch.object(
|
||||
module, "iter_moex_records", moex
|
||||
):
|
||||
return module.refresh_stock_exchange_activity(
|
||||
load_batch=batch, on_publish=on_publish
|
||||
)
|
||||
|
||||
def rows(self):
|
||||
return list(
|
||||
OrganizationSourceRecord.objects.filter(source="stock_exchange_activity")
|
||||
.order_by("external_id")
|
||||
.values("uid", "payload", "load_batch")
|
||||
)
|
||||
|
||||
def test_same_isin_on_two_exchanges_two_stable_records_no_new_org(self):
|
||||
artifact, result = self.run_import()
|
||||
before = self.rows()
|
||||
assert result.published == 2
|
||||
assert {r["payload"]["exchange"] for r in before} == {"moex", "spb"}
|
||||
assert len({r["uid"] for r in before}) == 2
|
||||
self.run_import(batch=2)
|
||||
assert [r["uid"] for r in self.rows()] == [r["uid"] for r in before]
|
||||
assert Organization.objects.count() == 1
|
||||
assert not OrganizationSourceRecord.objects.exclude(legacy_model="").exists()
|
||||
assert len(artifact.metadata["inputs"]) == 2
|
||||
assert {i["exchange"] for i in artifact.metadata["inputs"]} == {"moex", "spb"}
|
||||
assert all(i["checksum_sha256"] for i in artifact.metadata["inputs"])
|
||||
assert artifact.metadata["adapter_counters"]["spb"]["parsed"] == 1
|
||||
assert artifact.metadata["adapter_counters"]["moex"]["parsed"] == 1
|
||||
|
||||
def test_second_adapter_failure_keeps_combined_snapshot_and_two_inputs(self):
|
||||
self.run_import()
|
||||
before = self.rows()
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
self.run_import(batch=2, fail=True)
|
||||
assert self.rows() == before
|
||||
rejected = ParserSourceArtifact.objects.get(
|
||||
load_batch=2, original_name="exchange-manifest.json"
|
||||
)
|
||||
assert rejected.status == "rejected"
|
||||
assert len(rejected.metadata["inputs"]) == 2
|
||||
|
||||
def test_conflicting_vendor_identity_rejects_whole_snapshot(self):
|
||||
self.run_import()
|
||||
before = self.rows()
|
||||
with pytest.raises(SnapshotValidationError):
|
||||
self.run_import(batch=2, duplicates=True, count=2)
|
||||
assert self.rows() == before
|
||||
|
||||
def test_foreign_org_skipped_and_missing_inn_quarantined(self):
|
||||
artifact, result = self.run_import(inn="7810150688")
|
||||
assert result.skipped == 1
|
||||
assert result.published == 1
|
||||
artifact, result = self.run_import(batch=2, inn="")
|
||||
assert result.quarantined == 1
|
||||
assert artifact.metadata["adapter_counters"]["spb"]["invalid_issuer_inn"] == 1
|
||||
|
||||
def test_ambiguous_organization_quarantined_without_new_org(self):
|
||||
Organization.objects.create(
|
||||
name="Ambiguous",
|
||||
inn=self.org.inn,
|
||||
ogrn="1027700132194",
|
||||
okpo="01234567",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
artifact, result = self.run_import()
|
||||
assert result.quarantined == 2
|
||||
assert result.published == 0
|
||||
assert artifact.metadata["reasons"]["ambiguous_organization"] == 2
|
||||
|
||||
def test_invalid_canonical_requisites_never_published(self):
|
||||
self.org.ogrn = "bad"
|
||||
self.org.save()
|
||||
_, result = self.run_import()
|
||||
assert result.quarantined == 2
|
||||
assert result.published == 0
|
||||
|
||||
def test_staging_flushes_at_500(self):
|
||||
from apps.parsers.registry_snapshots import stage_records
|
||||
|
||||
sizes = []
|
||||
|
||||
def stage(rows):
|
||||
sizes.append(len(rows))
|
||||
stage_records(rows)
|
||||
|
||||
with patch.object(adapter(), "stage_records", stage):
|
||||
_, result = self.run_import(count=501)
|
||||
assert result.published == 502
|
||||
assert max(sizes) == 500
|
||||
|
||||
def test_publication_callback_failure_rolls_back(self):
|
||||
self.run_import()
|
||||
before = self.rows()
|
||||
|
||||
def fail(artifact, result):
|
||||
raise RuntimeError("synthetic callback failure")
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
self.run_import(batch=2, on_publish=fail)
|
||||
assert self.rows() == before
|
||||
Reference in New Issue
Block a user