"""A composed refresh exposes all parts together or preserves the old records.""" from unittest.mock import patch from uuid import uuid4 import pytest from apps.core.models import BackgroundJob from apps.parsers.aggregate_publication import finish_part from apps.parsers.models import ParserLoadLog, ParserRefreshRun, ParserRefreshStage from apps.parsers.source_cards import SourceCardService from organizations.models import Organization, OrganizationSourceRecord from organizations.source_ingestion import ( OrganizationSourceIngestionService, SourceRecordInput, ) @pytest.fixture def run(db): organization = Organization.objects.create( name="Factory", inn="7707083810", directory_imported_at="2026-10-09T00:00:00Z", opk_registry_membership=True, ) tasks = {"industrial": str(uuid4()), "manufactures": str(uuid4())} run = ParserRefreshRun.objects.create( source_card="manufacturers-and-products", tasks=tasks ) jobs = {} logs = {} for source, task_id in tasks.items(): jobs[source] = BackgroundJob.objects.create( task_id=task_id, task_name=f"test.{source}", meta={ "source": source, "aggregate_refresh_id": str(run.pk), "source_card": run.source_card, "refresh_task_ids": list(tasks.values()), }, ) jobs[source].mark_started() logs[source] = ParserLoadLog.objects.create( source=source, batch_id=1, status="in_progress" ) OrganizationSourceIngestionService.save_records( source=source, load_batch=0, records=[ SourceRecordInput( external_id="old", title="Old", organization_name=organization.name, inn=organization.inn, ) ], ) return run, jobs, logs, organization def stage(run, source, organization): return OrganizationSourceIngestionService.save_records( source=source, load_batch=1, publication_run_id=str(run.pk), records=[ SourceRecordInput( external_id="new", title="New", organization_name=organization.name, inn=organization.inn, payload={"issue_date": "09.10.2026"}, ) ], ) def ready(run, source, jobs, logs): return finish_part( jobs[source], logs[source], {"status": "success", "batch_id": 1, "saved": 1} ) def test_all_parts_and_terminal_jobs_are_published_together(run): run, jobs, logs, organization = run stage(run, "industrial", organization) ready(run, "industrial", jobs, logs) assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() jobs["industrial"].refresh_from_db() assert jobs["industrial"].status == "running" logs["industrial"].refresh_from_db() assert logs["industrial"].status == "in_progress" stage(run, "manufactures", organization) ready(run, "manufactures", jobs, logs) assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"} run.refresh_from_db() assert run.status == "published" assert not ParserRefreshStage.objects.exists() assert SourceCardService.get_card(run.source_card)["active_tasks"] == [] def test_part_failure_discards_staged_data_and_closes_all_jobs(run): run, jobs, logs, organization = run before = list( OrganizationSourceRecord.objects.values("uid", "title", "payload", "updated_at") ) stage(run, "industrial", organization) ready(run, "industrial", jobs, logs) jobs["manufactures"].fail("HTTPError: secret") assert ( list( OrganizationSourceRecord.objects.values( "uid", "title", "payload", "updated_at" ) ) == before ) assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"} assert not ParserRefreshStage.objects.exists() assert SourceCardService.get_card(run.source_card)["active_tasks"] == [] with pytest.raises(RuntimeError): stage(run, "manufactures", organization) def test_publication_failure_rolls_back_every_part(run): run, jobs, logs, organization = run stage(run, "industrial", organization) ready(run, "industrial", jobs, logs) stage(run, "manufactures", organization) original = OrganizationSourceIngestionService.save_records def save(**kwargs): if kwargs["source"] == "manufactures": raise RuntimeError("database publication failure") return original(**kwargs) with patch.object( OrganizationSourceIngestionService, "save_records", side_effect=save ), pytest.raises(RuntimeError): ready(run, "manufactures", jobs, logs) assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"} def test_cancellation_closes_siblings_and_late_delivery_cannot_publish(run): from apps.parsers.aggregate_publication import claim_part run, jobs, logs, organization = run stage(run, "industrial", organization) ready(run, "industrial", jobs, logs) jobs["manufactures"].revoke() assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"cancelled"} assert claim_part(jobs["manufactures"].task_id)["status"] == "cancelled" assert not ParserRefreshStage.objects.exists() assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() def test_duplicate_delivery_and_failure_after_commit_are_noops(run): from apps.parsers.aggregate_publication import claim_part run, jobs, logs, organization = run assert claim_part(jobs["industrial"].task_id) is None assert claim_part(jobs["industrial"].task_id)["duplicate_delivery"] is True for source in jobs: stage(run, source, organization) ready(run, source, jobs, logs) jobs["industrial"].fail("late failure") assert jobs["industrial"].status == "success" assert claim_part(jobs["industrial"].task_id)["status"] == "success" assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 def test_real_parser_tasks_stage_and_publish_through_the_barrier(run): from types import SimpleNamespace from apps.parsers.tasks import parse_industrial_production, parse_manufactures run, jobs, logs, organization = run certificate = SimpleNamespace( certificate_number="new", organisation_name=organization.name, inn=organization.inn, ogrn="", issue_date="09.10.2026", expiry_date="09.10.2027", certificate_file_url="", ) manufacturer = SimpleNamespace( inn=organization.inn, ogrn="", full_legal_name=organization.name, address="Factory address", ) with patch("apps.parsers.tasks.IndustrialProductionClient") as client: client.return_value.__enter__.return_value.fetch_certificates.return_value = [ certificate ] result = parse_industrial_production.apply( task_id=jobs["industrial"].task_id, kwargs={"proxies": []}, throw=True ).result assert result["publication_pending"] is True assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() with patch("apps.parsers.tasks.ManufacturesClient") as client: client.return_value.__enter__.return_value.fetch_manufacturers.return_value = [ manufacturer ] result = parse_manufactures.apply( task_id=jobs["manufactures"].task_id, kwargs={"proxies": []}, throw=True ).result assert result["status"] == "success" assert ( OrganizationSourceRecord.objects.filter( source="industrial", external_id="new" ).count() == 1 ) assert ( OrganizationSourceRecord.objects.filter( source="manufactures", external_id=organization.inn ).count() == 1 ) assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} @pytest.mark.django_db(transaction=True) def test_concurrent_last_parts_publish_once_on_postgres(run): from concurrent.futures import ThreadPoolExecutor from django.db import close_old_connections, connection if connection.vendor != "postgresql": pytest.skip("Requires PostgreSQL row locks and independent connections") run, jobs, logs, organization = run for source in jobs: stage(run, source, organization) def finish(source): close_old_connections() try: return finish_part( BackgroundJob.objects.get(pk=jobs[source].pk), ParserLoadLog.objects.get(pk=logs[source].pk), {"status": "success", "saved": 1, "batch_id": 1}, ) finally: close_old_connections() with ThreadPoolExecutor(max_workers=2) as workers: results = list(workers.map(finish, jobs)) assert [result["status"] for result in results].count("success") == 1 assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} @pytest.mark.django_db(transaction=True) def test_reader_sees_previous_snapshot_until_commit_on_postgres(run): from concurrent.futures import ThreadPoolExecutor from django.db import close_old_connections, connection if connection.vendor != "postgresql": pytest.skip("Requires independent PostgreSQL reader") run, jobs, logs, organization = run for source in jobs: stage(run, source, organization) ready(run, "industrial", jobs, logs) original = OrganizationSourceIngestionService.save_records def read(): close_old_connections() try: return OrganizationSourceRecord.objects.filter( external_id="new" ).count(), set(BackgroundJob.objects.values_list("status", flat=True)) finally: close_old_connections() def save(**kwargs): result = original(**kwargs) with ThreadPoolExecutor(max_workers=1) as reader: assert reader.submit(read).result(timeout=15) == (0, {"running"}) return result with patch.object( OrganizationSourceIngestionService, "save_records", side_effect=save ): ready(run, "manufactures", jobs, logs) assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 @pytest.mark.parametrize("fail_second", [False, True]) def test_defense_refresh_tasks_share_one_publication_boundary(db, fail_second): from types import SimpleNamespace from apps.parsers.clients.common import GenericParserItem from apps.parsers.tasks import _run_generic_parser organization = Organization.objects.create( name="Supplier", inn="7707083810", directory_imported_at="2026-10-09T00:00:00Z", opk_registry_membership=True, ) def enqueue(**kwargs): return {"task_id": kwargs["task_id"], "task_name": kwargs["task_name"]} with patch.object(SourceCardService, "_enqueue_task", side_effect=enqueue): SourceCardService._launch_refresh( SourceCardService.get_definition("defense-unreliable-suppliers"), requested_by_id=None, params={}, ) run = ParserRefreshRun.objects.get() for index, source in enumerate(run.tasks): def fetch(source=source, index=index): if fail_second and index == 1: raise RuntimeError("upstream failed") return [ GenericParserItem( source=source, external_id="new", inn=organization.inn, organisation_name=organization.name, ) ] job = BackgroundJob.objects.get(task_id=run.tasks[source]) def execute(job=job, source=source, fetch=fetch): return _run_generic_parser( SimpleNamespace(request=SimpleNamespace(id=job.task_id)), source_key=source, source=source, task_name=job.task_name, fetch_records=fetch, ) if fail_second and index == 1: with pytest.raises(RuntimeError): execute() else: execute() if index == 0: assert not OrganizationSourceRecord.objects.exists() assert OrganizationSourceRecord.objects.count() == (0 if fail_second else 2) assert set(BackgroundJob.objects.values_list("status", flat=True)) == ( {"failed"} if fail_second else {"success"} ) @pytest.mark.django_db(transaction=True) def test_cache_callback_failure_preserves_committed_success(run, caplog): run, jobs, logs, organization = run for source in jobs: stage(run, source, organization) ready(run, "industrial", jobs, logs) with patch( "apps.parsers.aggregate_publication.invalidate_source_data_cache", side_effect=RuntimeError("cache unavailable"), ), patch( "apps.parsers.tasks_registry_snapshots._refresh_published_exports" ) as export: ready(run, "manufactures", jobs, logs) export.assert_called_once() assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"} assert "cache invalidation failed" in caplog.text def test_second_composed_refresh_is_rejected_without_extra_jobs(db): from apps.core.exceptions import ConflictError definition = SourceCardService.get_definition("manufacturers-and-products") with patch.object(SourceCardService, "_enqueue_task", return_value={}): SourceCardService._launch_refresh(definition, requested_by_id=None, params={}) with pytest.raises(ConflictError): SourceCardService._launch_refresh( definition, requested_by_id=None, params={} ) assert ParserRefreshRun.objects.count() == 1 assert BackgroundJob.objects.count() == 3