Files
mostovik-backend/tests/apps/parsers/test_aggregate_publication.py
Aleksandr Meshchryakov 9b3da5b505
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m16s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
fix: implement backend testing amendments and atomic source publication
2026-10-12 09:47:28 +02:00

387 lines
14 KiB
Python

"""A composed refresh exposes all parts together or preserves the old records."""
from unittest.mock import patch
from uuid import uuid4
import pytest
from apps.core.models import BackgroundJob
from apps.parsers.aggregate_publication import finish_part
from apps.parsers.models import ParserLoadLog, ParserRefreshRun, ParserRefreshStage
from apps.parsers.source_cards import SourceCardService
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
@pytest.fixture
def run(db):
organization = Organization.objects.create(
name="Factory",
inn="7707083810",
directory_imported_at="2026-10-09T00:00:00Z",
opk_registry_membership=True,
)
tasks = {"industrial": str(uuid4()), "manufactures": str(uuid4())}
run = ParserRefreshRun.objects.create(
source_card="manufacturers-and-products", tasks=tasks
)
jobs = {}
logs = {}
for source, task_id in tasks.items():
jobs[source] = BackgroundJob.objects.create(
task_id=task_id,
task_name=f"test.{source}",
meta={
"source": source,
"aggregate_refresh_id": str(run.pk),
"source_card": run.source_card,
"refresh_task_ids": list(tasks.values()),
},
)
jobs[source].mark_started()
logs[source] = ParserLoadLog.objects.create(
source=source, batch_id=1, status="in_progress"
)
OrganizationSourceIngestionService.save_records(
source=source,
load_batch=0,
records=[
SourceRecordInput(
external_id="old",
title="Old",
organization_name=organization.name,
inn=organization.inn,
)
],
)
return run, jobs, logs, organization
def stage(run, source, organization):
return OrganizationSourceIngestionService.save_records(
source=source,
load_batch=1,
publication_run_id=str(run.pk),
records=[
SourceRecordInput(
external_id="new",
title="New",
organization_name=organization.name,
inn=organization.inn,
payload={"issue_date": "09.10.2026"},
)
],
)
def ready(run, source, jobs, logs):
return finish_part(
jobs[source], logs[source], {"status": "success", "batch_id": 1, "saved": 1}
)
def test_all_parts_and_terminal_jobs_are_published_together(run):
run, jobs, logs, organization = run
stage(run, "industrial", organization)
ready(run, "industrial", jobs, logs)
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
jobs["industrial"].refresh_from_db()
assert jobs["industrial"].status == "running"
logs["industrial"].refresh_from_db()
assert logs["industrial"].status == "in_progress"
stage(run, "manufactures", organization)
ready(run, "manufactures", jobs, logs)
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"}
run.refresh_from_db()
assert run.status == "published"
assert not ParserRefreshStage.objects.exists()
assert SourceCardService.get_card(run.source_card)["active_tasks"] == []
def test_part_failure_discards_staged_data_and_closes_all_jobs(run):
run, jobs, logs, organization = run
before = list(
OrganizationSourceRecord.objects.values("uid", "title", "payload", "updated_at")
)
stage(run, "industrial", organization)
ready(run, "industrial", jobs, logs)
jobs["manufactures"].fail("HTTPError: secret")
assert (
list(
OrganizationSourceRecord.objects.values(
"uid", "title", "payload", "updated_at"
)
)
== before
)
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"}
assert not ParserRefreshStage.objects.exists()
assert SourceCardService.get_card(run.source_card)["active_tasks"] == []
with pytest.raises(RuntimeError):
stage(run, "manufactures", organization)
def test_publication_failure_rolls_back_every_part(run):
run, jobs, logs, organization = run
stage(run, "industrial", organization)
ready(run, "industrial", jobs, logs)
stage(run, "manufactures", organization)
original = OrganizationSourceIngestionService.save_records
def save(**kwargs):
if kwargs["source"] == "manufactures":
raise RuntimeError("database publication failure")
return original(**kwargs)
with patch.object(
OrganizationSourceIngestionService, "save_records", side_effect=save
), pytest.raises(RuntimeError):
ready(run, "manufactures", jobs, logs)
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"}
def test_cancellation_closes_siblings_and_late_delivery_cannot_publish(run):
from apps.parsers.aggregate_publication import claim_part
run, jobs, logs, organization = run
stage(run, "industrial", organization)
ready(run, "industrial", jobs, logs)
jobs["manufactures"].revoke()
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"cancelled"}
assert claim_part(jobs["manufactures"].task_id)["status"] == "cancelled"
assert not ParserRefreshStage.objects.exists()
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
def test_duplicate_delivery_and_failure_after_commit_are_noops(run):
from apps.parsers.aggregate_publication import claim_part
run, jobs, logs, organization = run
assert claim_part(jobs["industrial"].task_id) is None
assert claim_part(jobs["industrial"].task_id)["duplicate_delivery"] is True
for source in jobs:
stage(run, source, organization)
ready(run, source, jobs, logs)
jobs["industrial"].fail("late failure")
assert jobs["industrial"].status == "success"
assert claim_part(jobs["industrial"].task_id)["status"] == "success"
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
def test_real_parser_tasks_stage_and_publish_through_the_barrier(run):
from types import SimpleNamespace
from apps.parsers.tasks import parse_industrial_production, parse_manufactures
run, jobs, logs, organization = run
certificate = SimpleNamespace(
certificate_number="new",
organisation_name=organization.name,
inn=organization.inn,
ogrn="",
issue_date="09.10.2026",
expiry_date="09.10.2027",
certificate_file_url="",
)
manufacturer = SimpleNamespace(
inn=organization.inn,
ogrn="",
full_legal_name=organization.name,
address="Factory address",
)
with patch("apps.parsers.tasks.IndustrialProductionClient") as client:
client.return_value.__enter__.return_value.fetch_certificates.return_value = [
certificate
]
result = parse_industrial_production.apply(
task_id=jobs["industrial"].task_id, kwargs={"proxies": []}, throw=True
).result
assert result["publication_pending"] is True
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
with patch("apps.parsers.tasks.ManufacturesClient") as client:
client.return_value.__enter__.return_value.fetch_manufacturers.return_value = [
manufacturer
]
result = parse_manufactures.apply(
task_id=jobs["manufactures"].task_id, kwargs={"proxies": []}, throw=True
).result
assert result["status"] == "success"
assert (
OrganizationSourceRecord.objects.filter(
source="industrial", external_id="new"
).count()
== 1
)
assert (
OrganizationSourceRecord.objects.filter(
source="manufactures", external_id=organization.inn
).count()
== 1
)
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
@pytest.mark.django_db(transaction=True)
def test_concurrent_last_parts_publish_once_on_postgres(run):
from concurrent.futures import ThreadPoolExecutor
from django.db import close_old_connections, connection
if connection.vendor != "postgresql":
pytest.skip("Requires PostgreSQL row locks and independent connections")
run, jobs, logs, organization = run
for source in jobs:
stage(run, source, organization)
def finish(source):
close_old_connections()
try:
return finish_part(
BackgroundJob.objects.get(pk=jobs[source].pk),
ParserLoadLog.objects.get(pk=logs[source].pk),
{"status": "success", "saved": 1, "batch_id": 1},
)
finally:
close_old_connections()
with ThreadPoolExecutor(max_workers=2) as workers:
results = list(workers.map(finish, jobs))
assert [result["status"] for result in results].count("success") == 1
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
@pytest.mark.django_db(transaction=True)
def test_reader_sees_previous_snapshot_until_commit_on_postgres(run):
from concurrent.futures import ThreadPoolExecutor
from django.db import close_old_connections, connection
if connection.vendor != "postgresql":
pytest.skip("Requires independent PostgreSQL reader")
run, jobs, logs, organization = run
for source in jobs:
stage(run, source, organization)
ready(run, "industrial", jobs, logs)
original = OrganizationSourceIngestionService.save_records
def read():
close_old_connections()
try:
return OrganizationSourceRecord.objects.filter(
external_id="new"
).count(), set(BackgroundJob.objects.values_list("status", flat=True))
finally:
close_old_connections()
def save(**kwargs):
result = original(**kwargs)
with ThreadPoolExecutor(max_workers=1) as reader:
assert reader.submit(read).result(timeout=15) == (0, {"running"})
return result
with patch.object(
OrganizationSourceIngestionService, "save_records", side_effect=save
):
ready(run, "manufactures", jobs, logs)
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
@pytest.mark.parametrize("fail_second", [False, True])
def test_defense_refresh_tasks_share_one_publication_boundary(db, fail_second):
from types import SimpleNamespace
from apps.parsers.clients.common import GenericParserItem
from apps.parsers.tasks import _run_generic_parser
organization = Organization.objects.create(
name="Supplier",
inn="7707083810",
directory_imported_at="2026-10-09T00:00:00Z",
opk_registry_membership=True,
)
def enqueue(**kwargs):
return {"task_id": kwargs["task_id"], "task_name": kwargs["task_name"]}
with patch.object(SourceCardService, "_enqueue_task", side_effect=enqueue):
SourceCardService._launch_refresh(
SourceCardService.get_definition("defense-unreliable-suppliers"),
requested_by_id=None,
params={},
)
run = ParserRefreshRun.objects.get()
for index, source in enumerate(run.tasks):
def fetch(source=source, index=index):
if fail_second and index == 1:
raise RuntimeError("upstream failed")
return [
GenericParserItem(
source=source,
external_id="new",
inn=organization.inn,
organisation_name=organization.name,
)
]
job = BackgroundJob.objects.get(task_id=run.tasks[source])
def execute(job=job, source=source, fetch=fetch):
return _run_generic_parser(
SimpleNamespace(request=SimpleNamespace(id=job.task_id)),
source_key=source,
source=source,
task_name=job.task_name,
fetch_records=fetch,
)
if fail_second and index == 1:
with pytest.raises(RuntimeError):
execute()
else:
execute()
if index == 0:
assert not OrganizationSourceRecord.objects.exists()
assert OrganizationSourceRecord.objects.count() == (0 if fail_second else 2)
assert set(BackgroundJob.objects.values_list("status", flat=True)) == (
{"failed"} if fail_second else {"success"}
)
@pytest.mark.django_db(transaction=True)
def test_cache_callback_failure_preserves_committed_success(run, caplog):
run, jobs, logs, organization = run
for source in jobs:
stage(run, source, organization)
ready(run, "industrial", jobs, logs)
with patch(
"apps.parsers.aggregate_publication.invalidate_source_data_cache",
side_effect=RuntimeError("cache unavailable"),
), patch(
"apps.parsers.tasks_registry_snapshots._refresh_published_exports"
) as export:
ready(run, "manufactures", jobs, logs)
export.assert_called_once()
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"}
assert "cache invalidation failed" in caplog.text
def test_second_composed_refresh_is_rejected_without_extra_jobs(db):
from apps.core.exceptions import ConflictError
definition = SourceCardService.get_definition("manufacturers-and-products")
with patch.object(SourceCardService, "_enqueue_task", return_value={}):
SourceCardService._launch_refresh(definition, requested_by_id=None, params={})
with pytest.raises(ConflictError):
SourceCardService._launch_refresh(
definition, requested_by_id=None, params={}
)
assert ParserRefreshRun.objects.count() == 1
assert BackgroundJob.objects.count() == 3