feat: publish unified exchange activity snapshots

This commit is contained in:
Aleksandr Meshchryakov
2026-10-05 17:14:10 +02:00
parent e2cf384853
commit d69ba600d4
8 changed files with 775 additions and 0 deletions

View File

@@ -0,0 +1,252 @@
"""Two private exchange inputs, one validated atomic canonical snapshot."""
from __future__ import annotations
import json
import re
from collections import Counter
from collections.abc import Callable
from io import BytesIO
from apps.parsers.exchange_moex import iter_moex_records
from apps.parsers.exchange_spb import (
EXCHANGE_SOURCE,
download_spb_artifact,
iter_spb_records,
)
from apps.parsers.fssp_csv import valid_legal_inn
from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord
from apps.parsers.registry_snapshots import (
SNAPSHOT_CHUNK_SIZE,
OwnOrganizationIndex,
SnapshotResult,
SnapshotValidationError,
publish_snapshot,
reject_snapshot,
save_artifact_file,
stage_records,
)
def _identity(row, index):
if not valid_legal_inn(row.inn):
return None, "invalid_issuer_inn"
organization, reason = index.resolve(inn=row.inn, ogrn="")
if not reason and (
not re.fullmatch(r"[0-9]{13}", organization.ogrn or "")
or not re.fullmatch(r"[0-9]{8}", organization.okpo or "")
):
return organization, "invalid_canonical_requisites"
return organization, reason
def _stage(
rows, *, exchange, raw, aggregate, index, counts, counters, pending, on_progress
):
row_number = aggregate.staged_records.count()
for row in rows:
counts["parsed"] += 1
if (
row.payload.get("exchange") != exchange
or not row.external_id.startswith(exchange + ":")
or row.record_type != "exchange_instrument"
or row.amount is not None
):
raise SnapshotValidationError("invalid_exchange_adapter_record")
organization, reason = _identity(row, index)
if reason:
counts[reason] += 1
counts["skipped" if reason == "outside_directory" else "quarantined"] += 1
if reason == "outside_directory":
continue
row.payload["provenance"] = {
**row.payload.get("provenance", {}),
"artifact_id": str(raw.uid),
"source_published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
}
row_number += 1
pending.append(
ParserStagedRecord(
artifact=aggregate,
row_number=row_number,
organization=organization,
external_id=row.external_id,
record_type=row.record_type,
raw_data=row.payload.get("upstream", {}),
normalized_data={
"title": row.title,
"record_date": row.record_date,
"amount": None,
"status": row.status,
"url": row.url,
"payload": row.payload,
},
disposition="quarantined" if reason else "staged",
reason=reason,
)
)
if len(pending) >= SNAPSHOT_CHUNK_SIZE:
stage_records(pending)
if on_progress:
on_progress(
sum(c["parsed"] for c in counters.values()),
0,
aggregate.staged_records.count(),
sum(c["quarantined"] for c in counters.values()),
)
stage_records(pending)
def _save_manifest(aggregate, raws, counters):
aggregate.parsed_count = sum(c["parsed"] for c in counters.values())
aggregate.quarantined_count = sum(c["quarantined"] for c in counters.values())
aggregate.metadata = {
**aggregate.metadata,
"inputs": [
{
"exchange": raw.metadata["dataset"],
"artifact_id": str(raw.uid),
"checksum_sha256": raw.sha256,
"size_bytes": raw.size_bytes,
"published_at": raw.source_published_at.isoformat()
if raw.source_published_at
else None,
"parsed_records_count": raw.parsed_count,
}
for raw in raws
],
"adapter_counters": {key: dict(value) for key, value in counters.items()},
"skipped_records_count": sum(c["skipped"] for c in counters.values()),
"reasons": dict(
sum(
(
Counter(
{
k: v
for k, v in c.items()
if k not in {"parsed", "skipped", "quarantined"}
}
)
for c in counters.values()
),
Counter(),
)
),
}
aggregate.rejection_reasons = aggregate.metadata["reasons"]
dates = [raw.source_published_at for raw in raws if raw.source_published_at]
aggregate.source_published_at = max(dates) if dates else None
save_artifact_file(
aggregate,
BytesIO(
json.dumps(aggregate.metadata, ensure_ascii=False, sort_keys=True).encode()
),
)
def refresh_stock_exchange_activity(
*,
load_batch: int,
uploaded_by_id: int | None = None,
on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
on_progress: Callable[[int, int, int, int], None] | None = None,
) -> tuple[ParserSourceArtifact, SnapshotResult]:
aggregate = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="exchange-manifest.json",
content_type="application/json",
version="spb-csv-v1+moex-iss-v1",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
)
raws = []
counters = {"spb": Counter(), "moex": Counter()}
pending = []
try:
index = OwnOrganizationIndex()
spb = download_spb_artifact(
load_batch=load_batch, uploaded_by_id=uploaded_by_id
)
raws.append(spb)
spb.metadata = {**spb.metadata, "aggregate_artifact_id": str(aggregate.uid)}
spb.save(update_fields=("metadata",))
with spb.file.open("rb") as handle:
_stage(
iter_spb_records(handle),
exchange="spb",
raw=spb,
aggregate=aggregate,
index=index,
counts=counters["spb"],
counters=counters,
pending=pending,
on_progress=on_progress,
)
spb.parsed_count = counters["spb"]["parsed"]
spb.status = "parsed"
spb.save()
moex = ParserSourceArtifact.objects.create(
source=EXCHANGE_SOURCE,
original_name="moex-responses.ndjson",
content_type="application/x-ndjson",
load_batch=load_batch,
uploaded_by_id=uploaded_by_id,
metadata={
"dataset": "moex",
"input_only": True,
"aggregate_artifact_id": str(aggregate.uid),
},
)
raws.append(moex)
_stage(
iter_moex_records(organization_index=index, artifact=moex),
exchange="moex",
raw=moex,
aggregate=aggregate,
index=index,
counts=counters["moex"],
counters=counters,
pending=pending,
on_progress=on_progress,
)
extra = moex.metadata.get("diagnostics", {})
counters["moex"]["parsed"] = moex.parsed_count
counters["moex"]["skipped"] += extra.get("outside_directory", 0)
counters["moex"]["quarantined"] += extra.get("missing_or_invalid_issuer_inn", 0)
counters["moex"]["missing_or_invalid_issuer_inn"] += extra.get(
"missing_or_invalid_issuer_inn", 0
)
# Raw checksum/date are final only once the generator reaches EOF.
for staged in aggregate.staged_records.all().iterator(
chunk_size=SNAPSHOT_CHUNK_SIZE
):
raw = (
spb if staged.normalized_data["payload"]["exchange"] == "spb" else moex
)
staged.normalized_data["payload"]["provenance"].update(
checksum_sha256=raw.sha256,
source_published_at=raw.source_published_at.isoformat()
if raw.source_published_at
else None,
)
staged.save(update_fields=("normalized_data",))
_save_manifest(aggregate, raws, counters)
aggregate.status = "parsed"
aggregate.save()
return aggregate, publish_snapshot(aggregate, on_publish=on_publish)
except Exception as exc:
# If discovery failed before returning the SPB input, retain its bounded diagnostics too.
if not raws:
raws = list(
ParserSourceArtifact.objects.filter(
source=EXCHANGE_SOURCE,
load_batch=load_batch,
metadata__input_only=True,
)
)
_save_manifest(aggregate, raws, counters)
for raw in raws:
reject_snapshot(raw, exc)
reject_snapshot(aggregate, exc)
raise

View File

@@ -0,0 +1,220 @@
# Generated by Django 3.2.25 on 2026-10-05 15:12
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("parsers", "0045_checko_disabled_schedules"),
]
operations = [
migrations.AlterField(
model_name="genericparserrecord",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
("hh", "Вакансии HeadHunter"),
("superjob", "Вакансии SuperJob"),
],
db_index=True,
help_text="Источник данных",
max_length=50,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parserbatchsequence",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
],
help_text="Источник данных",
max_length=50,
unique=True,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parserloadlog",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
],
db_index=True,
help_text="Источник данных",
max_length=50,
verbose_name="источник",
),
),
migrations.AlterField(
model_name="parsersourceartifact",
name="source",
field=models.CharField(
choices=[
("industrial", "Сертификаты промышленного производства"),
("industrial_products", "Реестр промышленной продукции"),
("manufactures", "Реестр производителей"),
("inspections", "Единый реестр проверок"),
("procurements", "Единая информационная система закупок"),
("fns_reports", "Бухгалтерская отчетность ФНС"),
("procurements_44fz", "Закупки 44-ФЗ"),
("procurements_223fz", "Закупки 223-ФЗ"),
("contracts", "Контракты ЕИС"),
("unfair_suppliers", "Недобросовестные поставщики"),
("fas_goz", "Уклонение от ГОЗ"),
("arbitration", "Арбитражные дела"),
("fedresurs_bankruptcy", "Банкротства Федресурс"),
("fstec", "Реестры ФСТЭК"),
("trudvsem", "Вакансии Работа России"),
(
"gosedo_address_directory",
"Глобальный адресный справочник ГосЭДО",
),
("media_news", "Новости СМИ"),
("ropk_sanctions", "РОПК — Санкции"),
(
"fns_sme_support_recipients",
"Реестр субъектов МСП — получателей поддержки",
),
("budget_ubpandnubp", "Реестр участников бюджетного процесса"),
("sro_membership_check", "Проверка членства в СРО"),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
(
"fssp_enforcement_proceedings",
"Исполнительные производства ФССП",
),
("stock_exchange_activity", "Биржевая активность"),
("checko_leasing", "Лизинг"),
("checko_messages", "Сообщения организаций"),
("checko_licenses", "Лицензии организаций"),
],
db_index=True,
max_length=50,
),
),
]

View File

@@ -66,6 +66,8 @@ class ParserLoadLog(TimestampMixin, models.Model):
_("Исполнительные производства ФССП"),
)
STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность")
CHECKO_LEASING = "checko_leasing", _("Лизинг")
CHECKO_MESSAGES = "checko_messages", _("Сообщения организаций")
CHECKO_LICENSES = "checko_licenses", _("Лицензии организаций")

View File

@@ -0,0 +1,74 @@
# Generated by Django 3.2.25 on 2026-10-05 15:12
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("organizations", "0017_checko_sections"),
]
operations = [
migrations.CreateModel(
name="StockExchangeActivityExtension",
fields=[
(
"organizationsourceextension_ptr",
models.OneToOneField(
auto_created=True,
on_delete=django.db.models.deletion.CASCADE,
parent_link=True,
primary_key=True,
serialize=False,
to="organizations.organizationsourceextension",
),
),
],
options={
"verbose_name": "биржевая активность",
"verbose_name_plural": "биржевая активность",
"db_table": "organizations_stock_exchange_activity_extension",
},
bases=("organizations.organizationsourceextension",),
),
migrations.AlterField(
model_name="organizationsourceextension",
name="source_group",
field=models.CharField(
choices=[
("financial_indicators", "Финансово-экономические показатели"),
("government_procurements", "Государственные закупки"),
("industrial_production", "Производители и продукция России"),
("planned_inspections", "Плановые проверки"),
("bankruptcy", "Сведения о процедурах банкротства"),
("defense_suppliers", "Недобросовестные поставщики ГОЗ"),
("arbitration", "Арбитражные дела"),
("security_registries", "Реестры по информационной безопасности"),
("vacancies", "Вакансии"),
("electronic_document_exchange", "Электронный документооборот"),
("media_mentions", "Упоминания в СМИ"),
("sanctions", "Санкции"),
("government_support", "Государственная поддержка"),
("sro_membership", "Членство в СРО"),
(
"budget_process_registry",
"Реестр участников бюджетного процесса",
),
(
"integrated_structure_documents",
"Документы интегрированных структур",
),
("fedresurs_company_profile", "Карточки компаний Федресурса"),
("enforcement_proceedings", "Исполнительные производства"),
("stock_exchange_activity", "Биржевая активность"),
("leasing", "Лизинг"),
("company_messages", "Сообщения организаций"),
("company_licenses", "Лицензии организаций"),
],
db_index=True,
max_length=64,
verbose_name="группа источников",
),
),
]

View File

@@ -57,6 +57,8 @@ class SourceGroup(models.TextChoices):
_("Исполнительные производства"),
)
STOCK_EXCHANGE_ACTIVITY = "stock_exchange_activity", _("Биржевая активность")
LEASING = "leasing", _("Лизинг")
COMPANY_MESSAGES = "company_messages", _("Сообщения организаций")
COMPANY_LICENSES = "company_licenses", _("Лицензии организаций")
@@ -1070,3 +1072,14 @@ class CompanyLicenseExtension(OrganizationSourceExtension):
db_table = "organizations_company_license_extension"
verbose_name = _("лицензии организации")
verbose_name_plural = _("лицензии организации")
class StockExchangeActivityExtension(OrganizationSourceExtension):
"""Combined SPB and MOEX observations for one canonical organization."""
source_group_value = SourceGroup.STOCK_EXCHANGE_ACTIVITY
class Meta:
db_table = "organizations_stock_exchange_activity_extension"
verbose_name = _("биржевая активность")
verbose_name_plural = _("биржевая активность")

View File

@@ -29,6 +29,7 @@ from organizations.models import (
SecurityRegistryExtension,
SourceGroup,
SroMembershipExtension,
StockExchangeActivityExtension,
VacancyExtension,
)
@@ -45,6 +46,13 @@ class SourceGroupDescriptor:
SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = {
ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY: SourceGroupDescriptor(
source=ParserLoadLog.Source.STOCK_EXCHANGE_ACTIVITY,
source_group=SourceGroup.STOCK_EXCHANGE_ACTIVITY,
record_type="exchange_instrument",
title="Биржевая активность",
extension_model=StockExchangeActivityExtension,
),
ParserLoadLog.Source.CHECKO_LICENSES: SourceGroupDescriptor(
source=ParserLoadLog.Source.CHECKO_LICENSES,
source_group=SourceGroup.COMPANY_LICENSES,

View File

@@ -528,3 +528,11 @@ SRO_REQUEST_INTERVAL_SECONDS = float(os.getenv("SRO_REQUEST_INTERVAL_SECONDS", "
SRO_HTTP_MAX_RESPONSE_BYTES = int(os.getenv("SRO_HTTP_MAX_RESPONSE_BYTES", "2097152"))
SRO_HTTP_TIMEOUT_SECONDS = int(os.getenv("SRO_HTTP_TIMEOUT_SECONDS", "30"))
SRO_MAX_REQUESTS_PER_RUN = int(os.getenv("SRO_MAX_REQUESTS_PER_RUN", "10000"))
# Public exchange collection bounds; TLS verification stays enabled.
EXCHANGE_SPB_MAX_BYTES = 50 * 1024**2
EXCHANGE_SPB_MAX_ROWS = 100000
EXCHANGE_MOEX_MAX_RESPONSE_BYTES = 10 * 1024**2
EXCHANGE_MOEX_MAX_PAGES = 1000
EXCHANGE_MOEX_MAX_REQUESTS = 10000
EXCHANGE_MOEX_MAX_DURATION_ROWS = 100000

View File

@@ -0,0 +1,198 @@
"""One source, two validated inputs, atomic publication with stable identities."""
import csv
import importlib
from io import BytesIO, StringIO
from pathlib import Path
from unittest.mock import patch
import pytest
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError, save_artifact_file
from django.test import TestCase, override_settings
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import SourceRecordInput
FIXTURES = Path(__file__).parents[2] / "fixtures" / "exchanges"
def adapter():
return importlib.import_module("apps.parsers.exchange_snapshot")
class ExchangeSnapshotTests(TestCase):
def setUp(self):
from tempfile import TemporaryDirectory
self.tmp = TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.settings = override_settings(
PARSER_PRIVATE_ARTIFACT_ROOT=self.tmp.name + "/private",
MEDIA_ROOT=self.tmp.name + "/public",
)
self.settings.enable()
self.addCleanup(self.settings.disable)
self.org = Organization.objects.create(
name="Canonical",
inn="7707083893",
ogrn="1027700132195",
okpo="00032537",
directory_imported_at=timezone.now(),
)
def run_import(
self, batch=1, fail=False, inn=None, duplicates=False, count=1, on_publish=None
):
module = adapter()
def download(**kwargs):
raw = ParserSourceArtifact.objects.create(
source="stock_exchange_activity",
original_name="spb.csv",
load_batch=batch,
source_published_at="2026-10-05",
metadata={"dataset": "spb", "input_only": True},
)
rows = list(
csv.reader(
StringIO(
(FIXTURES / "listing.csv").read_text(encoding="utf-8-sig")
),
delimiter=";",
)
)
rows[1][rows[0].index("e_INN_code")] = (
inn if inn is not None else self.org.inn
)
out = StringIO()
csv.writer(out, delimiter=";").writerows(rows)
save_artifact_file(raw, BytesIO(out.getvalue().encode()))
raw.refresh_from_db()
return raw
def moex(*, organization_index, artifact):
save_artifact_file(artifact, BytesIO(b'{"synthetic":true}\n'))
for n in range(count):
if fail:
raise SnapshotValidationError("synthetic_moex_failed")
yield SourceRecordInput(
external_id=f"moex:{0 if duplicates else n}",
record_type="exchange_instrument",
title="Synthetic bond",
organization_name=self.org.name,
inn=self.org.inn,
payload={
"exchange": "moex",
"instrument_id": str(n),
"issuer_inn": self.org.inn,
"isin": "RU0000000001",
"provenance": {},
"upstream": {"id": n},
},
)
artifact.parsed_count = count
artifact.source_published_at = timezone.now().date()
artifact.status = "parsed"
artifact.save()
with patch.object(module, "download_spb_artifact", download), patch.object(
module, "iter_moex_records", moex
):
return module.refresh_stock_exchange_activity(
load_batch=batch, on_publish=on_publish
)
def rows(self):
return list(
OrganizationSourceRecord.objects.filter(source="stock_exchange_activity")
.order_by("external_id")
.values("uid", "payload", "load_batch")
)
def test_same_isin_on_two_exchanges_two_stable_records_no_new_org(self):
artifact, result = self.run_import()
before = self.rows()
assert result.published == 2
assert {r["payload"]["exchange"] for r in before} == {"moex", "spb"}
assert len({r["uid"] for r in before}) == 2
self.run_import(batch=2)
assert [r["uid"] for r in self.rows()] == [r["uid"] for r in before]
assert Organization.objects.count() == 1
assert not OrganizationSourceRecord.objects.exclude(legacy_model="").exists()
assert len(artifact.metadata["inputs"]) == 2
assert {i["exchange"] for i in artifact.metadata["inputs"]} == {"moex", "spb"}
assert all(i["checksum_sha256"] for i in artifact.metadata["inputs"])
assert artifact.metadata["adapter_counters"]["spb"]["parsed"] == 1
assert artifact.metadata["adapter_counters"]["moex"]["parsed"] == 1
def test_second_adapter_failure_keeps_combined_snapshot_and_two_inputs(self):
self.run_import()
before = self.rows()
with pytest.raises(SnapshotValidationError):
self.run_import(batch=2, fail=True)
assert self.rows() == before
rejected = ParserSourceArtifact.objects.get(
load_batch=2, original_name="exchange-manifest.json"
)
assert rejected.status == "rejected"
assert len(rejected.metadata["inputs"]) == 2
def test_conflicting_vendor_identity_rejects_whole_snapshot(self):
self.run_import()
before = self.rows()
with pytest.raises(SnapshotValidationError):
self.run_import(batch=2, duplicates=True, count=2)
assert self.rows() == before
def test_foreign_org_skipped_and_missing_inn_quarantined(self):
artifact, result = self.run_import(inn="7810150688")
assert result.skipped == 1
assert result.published == 1
artifact, result = self.run_import(batch=2, inn="")
assert result.quarantined == 1
assert artifact.metadata["adapter_counters"]["spb"]["invalid_issuer_inn"] == 1
def test_ambiguous_organization_quarantined_without_new_org(self):
Organization.objects.create(
name="Ambiguous",
inn=self.org.inn,
ogrn="1027700132194",
okpo="01234567",
directory_imported_at=timezone.now(),
)
artifact, result = self.run_import()
assert result.quarantined == 2
assert result.published == 0
assert artifact.metadata["reasons"]["ambiguous_organization"] == 2
def test_invalid_canonical_requisites_never_published(self):
self.org.ogrn = "bad"
self.org.save()
_, result = self.run_import()
assert result.quarantined == 2
assert result.published == 0
def test_staging_flushes_at_500(self):
from apps.parsers.registry_snapshots import stage_records
sizes = []
def stage(rows):
sizes.append(len(rows))
stage_records(rows)
with patch.object(adapter(), "stage_records", stage):
_, result = self.run_import(count=501)
assert result.published == 502
assert max(sizes) == 500
def test_publication_callback_failure_rolls_back(self):
self.run_import()
before = self.rows()
def fail(artifact, result):
raise RuntimeError("synthetic callback failure")
with pytest.raises(RuntimeError):
self.run_import(batch=2, on_publish=fail)
assert self.rows() == before