Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m16s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
387 lines
14 KiB
Python
387 lines
14 KiB
Python
"""A composed refresh exposes all parts together or preserves the old records."""
|
|
from unittest.mock import patch
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from apps.core.models import BackgroundJob
|
|
from apps.parsers.aggregate_publication import finish_part
|
|
from apps.parsers.models import ParserLoadLog, ParserRefreshRun, ParserRefreshStage
|
|
from apps.parsers.source_cards import SourceCardService
|
|
from organizations.models import Organization, OrganizationSourceRecord
|
|
from organizations.source_ingestion import (
|
|
OrganizationSourceIngestionService,
|
|
SourceRecordInput,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def run(db):
|
|
organization = Organization.objects.create(
|
|
name="Factory",
|
|
inn="7707083810",
|
|
directory_imported_at="2026-10-09T00:00:00Z",
|
|
opk_registry_membership=True,
|
|
)
|
|
tasks = {"industrial": str(uuid4()), "manufactures": str(uuid4())}
|
|
run = ParserRefreshRun.objects.create(
|
|
source_card="manufacturers-and-products", tasks=tasks
|
|
)
|
|
jobs = {}
|
|
logs = {}
|
|
for source, task_id in tasks.items():
|
|
jobs[source] = BackgroundJob.objects.create(
|
|
task_id=task_id,
|
|
task_name=f"test.{source}",
|
|
meta={
|
|
"source": source,
|
|
"aggregate_refresh_id": str(run.pk),
|
|
"source_card": run.source_card,
|
|
"refresh_task_ids": list(tasks.values()),
|
|
},
|
|
)
|
|
jobs[source].mark_started()
|
|
logs[source] = ParserLoadLog.objects.create(
|
|
source=source, batch_id=1, status="in_progress"
|
|
)
|
|
OrganizationSourceIngestionService.save_records(
|
|
source=source,
|
|
load_batch=0,
|
|
records=[
|
|
SourceRecordInput(
|
|
external_id="old",
|
|
title="Old",
|
|
organization_name=organization.name,
|
|
inn=organization.inn,
|
|
)
|
|
],
|
|
)
|
|
return run, jobs, logs, organization
|
|
|
|
|
|
def stage(run, source, organization):
|
|
return OrganizationSourceIngestionService.save_records(
|
|
source=source,
|
|
load_batch=1,
|
|
publication_run_id=str(run.pk),
|
|
records=[
|
|
SourceRecordInput(
|
|
external_id="new",
|
|
title="New",
|
|
organization_name=organization.name,
|
|
inn=organization.inn,
|
|
payload={"issue_date": "09.10.2026"},
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
def ready(run, source, jobs, logs):
|
|
return finish_part(
|
|
jobs[source], logs[source], {"status": "success", "batch_id": 1, "saved": 1}
|
|
)
|
|
|
|
|
|
def test_all_parts_and_terminal_jobs_are_published_together(run):
|
|
run, jobs, logs, organization = run
|
|
stage(run, "industrial", organization)
|
|
ready(run, "industrial", jobs, logs)
|
|
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
|
|
jobs["industrial"].refresh_from_db()
|
|
assert jobs["industrial"].status == "running"
|
|
logs["industrial"].refresh_from_db()
|
|
assert logs["industrial"].status == "in_progress"
|
|
stage(run, "manufactures", organization)
|
|
ready(run, "manufactures", jobs, logs)
|
|
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
|
|
assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"}
|
|
run.refresh_from_db()
|
|
assert run.status == "published"
|
|
assert not ParserRefreshStage.objects.exists()
|
|
assert SourceCardService.get_card(run.source_card)["active_tasks"] == []
|
|
|
|
|
|
def test_part_failure_discards_staged_data_and_closes_all_jobs(run):
|
|
run, jobs, logs, organization = run
|
|
before = list(
|
|
OrganizationSourceRecord.objects.values("uid", "title", "payload", "updated_at")
|
|
)
|
|
stage(run, "industrial", organization)
|
|
ready(run, "industrial", jobs, logs)
|
|
jobs["manufactures"].fail("HTTPError: secret")
|
|
assert (
|
|
list(
|
|
OrganizationSourceRecord.objects.values(
|
|
"uid", "title", "payload", "updated_at"
|
|
)
|
|
)
|
|
== before
|
|
)
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"}
|
|
assert not ParserRefreshStage.objects.exists()
|
|
assert SourceCardService.get_card(run.source_card)["active_tasks"] == []
|
|
with pytest.raises(RuntimeError):
|
|
stage(run, "manufactures", organization)
|
|
|
|
|
|
def test_publication_failure_rolls_back_every_part(run):
|
|
run, jobs, logs, organization = run
|
|
stage(run, "industrial", organization)
|
|
ready(run, "industrial", jobs, logs)
|
|
stage(run, "manufactures", organization)
|
|
original = OrganizationSourceIngestionService.save_records
|
|
|
|
def save(**kwargs):
|
|
if kwargs["source"] == "manufactures":
|
|
raise RuntimeError("database publication failure")
|
|
return original(**kwargs)
|
|
|
|
with patch.object(
|
|
OrganizationSourceIngestionService, "save_records", side_effect=save
|
|
), pytest.raises(RuntimeError):
|
|
ready(run, "manufactures", jobs, logs)
|
|
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"}
|
|
|
|
|
|
def test_cancellation_closes_siblings_and_late_delivery_cannot_publish(run):
|
|
from apps.parsers.aggregate_publication import claim_part
|
|
|
|
run, jobs, logs, organization = run
|
|
stage(run, "industrial", organization)
|
|
ready(run, "industrial", jobs, logs)
|
|
jobs["manufactures"].revoke()
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"cancelled"}
|
|
assert claim_part(jobs["manufactures"].task_id)["status"] == "cancelled"
|
|
assert not ParserRefreshStage.objects.exists()
|
|
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
|
|
|
|
|
|
def test_duplicate_delivery_and_failure_after_commit_are_noops(run):
|
|
from apps.parsers.aggregate_publication import claim_part
|
|
|
|
run, jobs, logs, organization = run
|
|
assert claim_part(jobs["industrial"].task_id) is None
|
|
assert claim_part(jobs["industrial"].task_id)["duplicate_delivery"] is True
|
|
for source in jobs:
|
|
stage(run, source, organization)
|
|
ready(run, source, jobs, logs)
|
|
jobs["industrial"].fail("late failure")
|
|
assert jobs["industrial"].status == "success"
|
|
assert claim_part(jobs["industrial"].task_id)["status"] == "success"
|
|
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
|
|
|
|
|
|
def test_real_parser_tasks_stage_and_publish_through_the_barrier(run):
|
|
from types import SimpleNamespace
|
|
|
|
from apps.parsers.tasks import parse_industrial_production, parse_manufactures
|
|
|
|
run, jobs, logs, organization = run
|
|
certificate = SimpleNamespace(
|
|
certificate_number="new",
|
|
organisation_name=organization.name,
|
|
inn=organization.inn,
|
|
ogrn="",
|
|
issue_date="09.10.2026",
|
|
expiry_date="09.10.2027",
|
|
certificate_file_url="",
|
|
)
|
|
manufacturer = SimpleNamespace(
|
|
inn=organization.inn,
|
|
ogrn="",
|
|
full_legal_name=organization.name,
|
|
address="Factory address",
|
|
)
|
|
with patch("apps.parsers.tasks.IndustrialProductionClient") as client:
|
|
client.return_value.__enter__.return_value.fetch_certificates.return_value = [
|
|
certificate
|
|
]
|
|
result = parse_industrial_production.apply(
|
|
task_id=jobs["industrial"].task_id, kwargs={"proxies": []}, throw=True
|
|
).result
|
|
assert result["publication_pending"] is True
|
|
assert not OrganizationSourceRecord.objects.filter(external_id="new").exists()
|
|
with patch("apps.parsers.tasks.ManufacturesClient") as client:
|
|
client.return_value.__enter__.return_value.fetch_manufacturers.return_value = [
|
|
manufacturer
|
|
]
|
|
result = parse_manufactures.apply(
|
|
task_id=jobs["manufactures"].task_id, kwargs={"proxies": []}, throw=True
|
|
).result
|
|
assert result["status"] == "success"
|
|
assert (
|
|
OrganizationSourceRecord.objects.filter(
|
|
source="industrial", external_id="new"
|
|
).count()
|
|
== 1
|
|
)
|
|
assert (
|
|
OrganizationSourceRecord.objects.filter(
|
|
source="manufactures", external_id=organization.inn
|
|
).count()
|
|
== 1
|
|
)
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
|
|
|
|
|
|
@pytest.mark.django_db(transaction=True)
|
|
def test_concurrent_last_parts_publish_once_on_postgres(run):
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from django.db import close_old_connections, connection
|
|
|
|
if connection.vendor != "postgresql":
|
|
pytest.skip("Requires PostgreSQL row locks and independent connections")
|
|
run, jobs, logs, organization = run
|
|
for source in jobs:
|
|
stage(run, source, organization)
|
|
|
|
def finish(source):
|
|
close_old_connections()
|
|
try:
|
|
return finish_part(
|
|
BackgroundJob.objects.get(pk=jobs[source].pk),
|
|
ParserLoadLog.objects.get(pk=logs[source].pk),
|
|
{"status": "success", "saved": 1, "batch_id": 1},
|
|
)
|
|
finally:
|
|
close_old_connections()
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as workers:
|
|
results = list(workers.map(finish, jobs))
|
|
assert [result["status"] for result in results].count("success") == 1
|
|
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
|
|
|
|
|
|
@pytest.mark.django_db(transaction=True)
|
|
def test_reader_sees_previous_snapshot_until_commit_on_postgres(run):
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from django.db import close_old_connections, connection
|
|
|
|
if connection.vendor != "postgresql":
|
|
pytest.skip("Requires independent PostgreSQL reader")
|
|
run, jobs, logs, organization = run
|
|
for source in jobs:
|
|
stage(run, source, organization)
|
|
ready(run, "industrial", jobs, logs)
|
|
original = OrganizationSourceIngestionService.save_records
|
|
|
|
def read():
|
|
close_old_connections()
|
|
try:
|
|
return OrganizationSourceRecord.objects.filter(
|
|
external_id="new"
|
|
).count(), set(BackgroundJob.objects.values_list("status", flat=True))
|
|
finally:
|
|
close_old_connections()
|
|
|
|
def save(**kwargs):
|
|
result = original(**kwargs)
|
|
with ThreadPoolExecutor(max_workers=1) as reader:
|
|
assert reader.submit(read).result(timeout=15) == (0, {"running"})
|
|
return result
|
|
|
|
with patch.object(
|
|
OrganizationSourceIngestionService, "save_records", side_effect=save
|
|
):
|
|
ready(run, "manufactures", jobs, logs)
|
|
assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2
|
|
|
|
|
|
@pytest.mark.parametrize("fail_second", [False, True])
|
|
def test_defense_refresh_tasks_share_one_publication_boundary(db, fail_second):
|
|
from types import SimpleNamespace
|
|
|
|
from apps.parsers.clients.common import GenericParserItem
|
|
from apps.parsers.tasks import _run_generic_parser
|
|
|
|
organization = Organization.objects.create(
|
|
name="Supplier",
|
|
inn="7707083810",
|
|
directory_imported_at="2026-10-09T00:00:00Z",
|
|
opk_registry_membership=True,
|
|
)
|
|
|
|
def enqueue(**kwargs):
|
|
return {"task_id": kwargs["task_id"], "task_name": kwargs["task_name"]}
|
|
|
|
with patch.object(SourceCardService, "_enqueue_task", side_effect=enqueue):
|
|
SourceCardService._launch_refresh(
|
|
SourceCardService.get_definition("defense-unreliable-suppliers"),
|
|
requested_by_id=None,
|
|
params={},
|
|
)
|
|
run = ParserRefreshRun.objects.get()
|
|
for index, source in enumerate(run.tasks):
|
|
|
|
def fetch(source=source, index=index):
|
|
if fail_second and index == 1:
|
|
raise RuntimeError("upstream failed")
|
|
return [
|
|
GenericParserItem(
|
|
source=source,
|
|
external_id="new",
|
|
inn=organization.inn,
|
|
organisation_name=organization.name,
|
|
)
|
|
]
|
|
|
|
job = BackgroundJob.objects.get(task_id=run.tasks[source])
|
|
|
|
def execute(job=job, source=source, fetch=fetch):
|
|
return _run_generic_parser(
|
|
SimpleNamespace(request=SimpleNamespace(id=job.task_id)),
|
|
source_key=source,
|
|
source=source,
|
|
task_name=job.task_name,
|
|
fetch_records=fetch,
|
|
)
|
|
|
|
if fail_second and index == 1:
|
|
with pytest.raises(RuntimeError):
|
|
execute()
|
|
else:
|
|
execute()
|
|
if index == 0:
|
|
assert not OrganizationSourceRecord.objects.exists()
|
|
assert OrganizationSourceRecord.objects.count() == (0 if fail_second else 2)
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == (
|
|
{"failed"} if fail_second else {"success"}
|
|
)
|
|
|
|
|
|
@pytest.mark.django_db(transaction=True)
|
|
def test_cache_callback_failure_preserves_committed_success(run, caplog):
|
|
run, jobs, logs, organization = run
|
|
for source in jobs:
|
|
stage(run, source, organization)
|
|
ready(run, "industrial", jobs, logs)
|
|
with patch(
|
|
"apps.parsers.aggregate_publication.invalidate_source_data_cache",
|
|
side_effect=RuntimeError("cache unavailable"),
|
|
), patch(
|
|
"apps.parsers.tasks_registry_snapshots._refresh_published_exports"
|
|
) as export:
|
|
ready(run, "manufactures", jobs, logs)
|
|
export.assert_called_once()
|
|
assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"}
|
|
assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"}
|
|
assert "cache invalidation failed" in caplog.text
|
|
|
|
|
|
def test_second_composed_refresh_is_rejected_without_extra_jobs(db):
|
|
from apps.core.exceptions import ConflictError
|
|
|
|
definition = SourceCardService.get_definition("manufacturers-and-products")
|
|
with patch.object(SourceCardService, "_enqueue_task", return_value={}):
|
|
SourceCardService._launch_refresh(definition, requested_by_id=None, params={})
|
|
with pytest.raises(ConflictError):
|
|
SourceCardService._launch_refresh(
|
|
definition, requested_by_id=None, params={}
|
|
)
|
|
assert ParserRefreshRun.objects.count() == 1
|
|
assert BackgroundJob.objects.count() == 3
|