feat(registries): add SME and budget imports and fix source API workflows

tests/apps/parsers/test_refresh_progress.py

@@ -0,0 +1,147 @@
+"""Общий прогресс ручного запуска, в том числе после завершения его частей."""
+
+from datetime import timedelta
+from unittest.mock import MagicMock
+
+import pytest
+from apps.core.models import BackgroundJob, JobStatus
+from apps.parsers.source_cards import SourceCardService
+from django.test import override_settings
+from django.utils import timezone
+
+
+@pytest.mark.django_db
+def test_all_jobs_are_visible_before_dispatch_and_completed_parts_stay_in_progress():
+    definition = SourceCardService.get_definition("manufacturers-and-products")
+    observed_counts = []
+    tasks = [MagicMock(), MagicMock(), MagicMock()]
+
+    def dispatch(**kwargs):
+        jobs = list(BackgroundJob.objects.all())
+        observed_counts.append(len(jobs))
+        assert {job.task_id for job in jobs} == set(jobs[0].meta["refresh_task_ids"])
+
+    for task in tasks:
+        task.apply_async.side_effect = dispatch
+    specs = tuple(
+        (task, name, source)
+        for task, name, source in zip(
+            tasks,
+            [
+                "apps.parsers.tasks.parse_industrial_production",
+                "apps.parsers.tasks.parse_industrial_products",
+                "apps.parsers.tasks.parse_manufactures",
+            ],
+            ["industrial", "industrial_products", "manufactures"],
+            strict=False,
+        )
+    )
+    result = SourceCardService._enqueue_refresh_group(
+        definition, specs, requested_by_id=1, kwargs={}
+    )
+    assert observed_counts == [3, 3, 3]
+    jobs = [BackgroundJob.objects.get(task_id=item["task_id"]) for item in result]
+    jobs[0].complete()
+    jobs[1].update_progress(50)
+    jobs[2].update_progress(0)
+    card = SourceCardService.get_card(definition.slug)
+    assert card["progress"] == 50
+    assert len(card["active_tasks"]) == 2
+    jobs[1].complete()
+    assert SourceCardService.get_card(definition.slug)["progress"] == 67
+    jobs[2].complete()
+    for card in [
+        SourceCardService.get_card(definition.slug),
+        next(
+            card
+            for card in SourceCardService.list_cards()
+            if card["slug"] == definition.slug
+        ),
+    ]:
+        assert (card["progress"], card["status"], card["active_tasks"]) == (
+            100,
+            "success",
+            [],
+        )
+
+
+@pytest.mark.django_db
+def test_failed_run_keeps_progress_and_error_after_last_active_job_disappears():
+    job = BackgroundJob.objects.create(
+        task_id="failed-refresh",
+        task_name="apps.parsers.tasks.parse_arbitration_cases",
+        meta={
+            "source_card": "arbitration-cases",
+            "refresh_task_ids": ["failed-refresh"],
+        },
+    )
+    job.update_progress(60)
+    job.fail("Ошибка загрузки")
+    card = SourceCardService.get_card("arbitration-cases")
+    assert (card["status"], card["progress"], card["error_message"]) == (
+        "error",
+        60,
+        "Ошибка загрузки",
+    )
+
+
+@pytest.mark.django_db
+def test_broker_failure_keeps_all_registered_parts_in_final_failed_state():
+    definition = SourceCardService.get_definition("defense-unreliable-suppliers")
+    task = MagicMock()
+    task.apply_async.side_effect = RuntimeError("broker unavailable")
+    with pytest.raises(RuntimeError):
+        SourceCardService._enqueue_refresh_group(
+            definition,
+            [
+                (task, "test.first", "unfair_suppliers"),
+                (task, "test.second", "fas_goz"),
+            ],
+            requested_by_id=1,
+            kwargs={},
+        )
+    jobs = list(BackgroundJob.objects.all())
+    assert len(jobs) == 2
+    assert all(job.status == JobStatus.FAILURE and job.error for job in jobs)
+    assert task.apply_async.call_count == 1
+
+
+@pytest.mark.django_db
+def test_completed_new_run_does_not_hide_active_older_run():
+    definition = SourceCardService.get_definition("manufacturers-and-products")
+    specs = [(MagicMock(), name, "industrial") for name in definition.task_names]
+    first = SourceCardService._enqueue_refresh_group(
+        definition, specs, requested_by_id=None, kwargs={}
+    )
+    second = SourceCardService._enqueue_refresh_group(
+        definition, specs, requested_by_id=None, kwargs={}
+    )
+    for item in second:
+        BackgroundJob.objects.get(task_id=item["task_id"]).complete()
+    card = SourceCardService.get_card(definition.slug)
+    assert {job["task_id"] for job in card["active_tasks"]} == {
+        job["task_id"] for job in first
+    }
+    assert (card["status"], card["progress"]) == ("in_progress", 50)
+    for item in first:
+        BackgroundJob.objects.get(task_id=item["task_id"]).complete()
+    assert SourceCardService.get_card(definition.slug)["progress"] == 100
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "status,time_field",
+    [(JobStatus.PENDING, "created_at"), (JobStatus.STARTED, "updated_at")],
+)
+@override_settings(
+    PARSER_STALE_LOAD_MAX_AGE_MINUTES=10, PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES=10
+)
+def test_latest_task_does_not_reactivate_a_stale_job(status, time_field):
+    definition = SourceCardService.get_definition("arbitration-cases")
+    job = BackgroundJob.objects.create(
+        task_id="stale-task", task_name=definition.task_names[0], status=status
+    )
+    BackgroundJob.objects.filter(pk=job.pk).update(
+        **{time_field: timezone.now() - timedelta(minutes=15)}
+    )
+    assert SourceCardService.get_card(definition.slug)["active_tasks"] == []