feat(registries): add SME and budget imports and fix source API workflows

tests/apps/parsers/test_vacancy_progress_race.py

@@ -0,0 +1,113 @@
+"""Запоздалые контрольные точки вакансий не меняют актуальное состояние задачи."""
+
+from unittest.mock import patch
+
+import pytest
+from apps.core.models import BackgroundJob
+from apps.parsers.models import ParserLoadLog
+from apps.parsers.tasks import (
+    _find_resumable_vacancy_job,
+    _update_vacancy_registry_checkpoint,
+)
+
+
+def _checkpoint(job, load_log, *, processed):
+    with patch("apps.parsers.tasks._vacancy_batch_records_count", return_value=17):
+        return _update_vacancy_registry_checkpoint(
+            job=job,
+            load_log=load_log,
+            batch_id=load_log.batch_id,
+            processed=processed,
+            total=100,
+            failed=0,
+            targets_signature="test-targets",
+        )
+
+
+@pytest.fixture
+def vacancy_run(db):
+    job = BackgroundJob.objects.create(
+        task_id="vacancy-checkpoint",
+        task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
+        meta={"source_card": "labor-vacancies"},
+    )
+    load_log = ParserLoadLog.objects.create(
+        source=ParserLoadLog.Source.TRUDVSEM,
+        batch_id=1,
+        status=ParserLoadLog.Status.IN_PROGRESS,
+    )
+    return job, load_log
+
+
+def test_lower_stale_vacancy_checkpoint_preserves_progress_message_and_meta(
+    vacancy_run,
+):
+    job, load_log = vacancy_run
+    stale = BackgroundJob.objects.get(pk=job.pk)
+    assert _checkpoint(job, load_log, processed=75) == 17
+    expected = (job.progress, job.progress_message, job.meta)
+
+    assert _checkpoint(stale, load_log, processed=25) == 17
+
+    job.refresh_from_db()
+    assert (job.progress, job.progress_message, job.meta) == expected
+    assert job.meta["next_offset"] == 75
+    assert job.meta["source_card"] == "labor-vacancies"
+    load_log.refresh_from_db()
+    assert load_log.records_count == 17
+    assert load_log.status == ParserLoadLog.Status.IN_PROGRESS
+
+
+@pytest.mark.parametrize("terminal", ["complete", "fail", "revoke"])
+def test_terminal_vacancy_job_ignores_late_checkpoint(vacancy_run, terminal):
+    job, load_log = vacancy_run
+    _checkpoint(job, load_log, processed=65)
+    stale = BackgroundJob.objects.get(pk=job.pk)
+    if terminal == "fail":
+        job.fail("Ошибка источника")
+    else:
+        getattr(job, terminal)()
+    expected = (
+        job.status,
+        job.progress,
+        job.progress_message,
+        job.meta,
+        job.completed_at,
+    )
+
+    assert _checkpoint(stale, load_log, processed=90) == 17
+
+    job.refresh_from_db()
+    assert (
+        job.status,
+        job.progress,
+        job.progress_message,
+        job.meta,
+        job.completed_at,
+    ) == expected
+
+
+@pytest.mark.parametrize("saved_records", [0, 17])
+def test_zero_cursor_resumes_only_when_batch_has_saved_records(
+    vacancy_run, saved_records
+):
+    job, load_log = vacancy_run
+    job.meta.update(
+        batch_id=load_log.batch_id,
+        next_offset=0,
+        total_organizations=100,
+        targets_signature="test-targets",
+    )
+    job.save(update_fields=["meta"])
+    job.revoke()
+
+    with patch(
+        "apps.parsers.tasks._vacancy_batch_records_count", return_value=saved_records
+    ):
+        result = _find_resumable_vacancy_job(
+            exclude_task_id="new-run",
+            targets_signature="test-targets",
+            targets_count=100,
+        )
+
+    assert result == ((job, load_log) if saved_records else (None, None))