fix(parsers): close stale source jobs

tests/apps/core/test_background_jobs.py

@@ -1,8 +1,11 @@
 """Тесты для BackgroundJob."""
 
+from datetime import timedelta
+
 from apps.core.models import BackgroundJob, JobStatus
 from apps.core.services import BackgroundJobService
 from django.test import TestCase
+from django.utils import timezone
 from faker import Faker
 
 fake = Faker()
@@ -252,10 +255,6 @@ class BackgroundJobServiceTest(TestCase):
         self.assertEqual([j.task_id for j in active_jobs], [job_user.task_id])
 
     def test_cleanup_old_jobs(self):
-        from datetime import timedelta
-
-        from django.utils import timezone
-
         old_job = BackgroundJobService.create_job(
             task_id="job-old",
             task_name="test.task",
@@ -272,3 +271,39 @@ class BackgroundJobServiceTest(TestCase):
 
         deleted = BackgroundJobService.cleanup_old_jobs(days=30)
         self.assertEqual(deleted, 1)
+
+    def test_mark_stale_active_jobs_failed_scopes_by_task_and_source(self):
+        stale_job = BackgroundJobService.create_job(
+            task_id="job-stale",
+            task_name="apps.parsers.tasks.parse_industrial_products",
+            meta={"source": "industrial_products"},
+        )
+        fresh_job = BackgroundJobService.create_job(
+            task_id="job-fresh",
+            task_name="apps.parsers.tasks.parse_industrial_products",
+            meta={"source": "industrial_products"},
+        )
+        unrelated_job = BackgroundJobService.create_job(
+            task_id="job-unrelated",
+            task_name="apps.other.tasks.task",
+            meta={"source": "industrial_products"},
+        )
+        old_timestamp = timezone.now() - timedelta(hours=3)
+        BackgroundJob.objects.filter(
+            task_id__in=[stale_job.task_id, unrelated_job.task_id]
+        ).update(updated_at=old_timestamp)
+
+        updated = BackgroundJobService.mark_stale_active_jobs_failed(
+            max_age_minutes=90,
+            task_names={"apps.parsers.tasks.parse_industrial_products"},
+            meta_sources={"industrial_products"},
+        )
+
+        stale_job.refresh_from_db()
+        fresh_job.refresh_from_db()
+        unrelated_job.refresh_from_db()
+        self.assertEqual(updated, 1)
+        self.assertEqual(stale_job.status, JobStatus.FAILURE)
+        self.assertIn("Stale background job", stale_job.error)
+        self.assertEqual(fresh_job.status, JobStatus.PENDING)
+        self.assertEqual(unrelated_job.status, JobStatus.PENDING)

tests/apps/parsers/test_services.py

@@ -411,6 +411,31 @@ class ParserLoadLogServiceTest(TestCase):
         self.assertEqual(updated, 0)
         self.assertEqual(log.status, ParserLoadLog.Status.IN_PROGRESS)
 
+    def test_mark_stale_in_progress_failed_closes_precreated_job_without_batch(self):
+        """Pre-created source-card jobs without batch_id are still linked by source."""
+        log = ParserLoadLogFactory(
+            source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS,
+            batch_id=2,
+            status=ParserLoadLog.Status.IN_PROGRESS,
+        )
+        job = BackgroundJob.objects.create(
+            task_id="precreated-source-card-task",
+            task_name="apps.parsers.tasks.parse_industrial_products",
+            status=JobStatus.STARTED,
+            meta={"source": log.source, "source_card": "manufacturers-and-products"},
+        )
+        old_timestamp = timezone.now() - timedelta(hours=3)
+        ParserLoadLog.objects.filter(pk=log.pk).update(updated_at=old_timestamp)
+        BackgroundJob.objects.filter(pk=job.pk).update(updated_at=old_timestamp)
+
+        updated = ParserLoadLogService.mark_stale_in_progress_failed(max_age_minutes=90)
+
+        log.refresh_from_db()
+        job.refresh_from_db()
+        self.assertEqual(updated, 1)
+        self.assertEqual(log.status, ParserLoadLog.Status.FAILED)
+        self.assertEqual(job.status, JobStatus.FAILURE)
+
 
 class IndustrialCertificateServiceTest(TestCase):
     """Tests for IndustrialCertificateService."""

tests/apps/parsers/test_source_cards_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from datetime import timedelta
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 
@@ -10,6 +11,7 @@ from apps.parsers.source_cards import (
 )
 from django.http import Http404
 from django.test import SimpleTestCase
+from django.utils import timezone
 from rest_framework.exceptions import ValidationError
 
 
@@ -249,6 +251,19 @@ class SourceCardServiceUnitTest(SimpleTestCase):
             ),
             "in_progress",
         )
+        stale_in_progress_load = SimpleNamespace(
+            status="in_progress",
+            updated_at=timezone.now() - timedelta(hours=3),
+        )
+        self.assertEqual(
+            SourceCardService._get_status(
+                definition=SourceCardService.get_definition("financial-indicators"),
+                active_tasks=[],
+                latest_load=stale_in_progress_load,
+                last_updated_at=None,
+            ),
+            "error",
+        )
         self.assertEqual(
             SourceCardService._get_status(
                 definition=SourceCardService.get_definition("financial-indicators"),

tests/apps/parsers/test_tasks.py

@@ -12,6 +12,7 @@ from types import SimpleNamespace
 from unittest.mock import patch
 from urllib.parse import urlparse
 
+from apps.core.services import BackgroundJobService
 from apps.parsers import tasks as parser_tasks
 from apps.parsers.clients.base import HTTPError
 from apps.parsers.clients.minpromtorg.industrial import (
@@ -285,14 +286,46 @@ class GenericSourceFetchTestCase(TestCase):
             ParserLoadLogService,
             "mark_stale_in_progress_failed",
             return_value=2,
-        ) as cleanup_mock:
+        ) as cleanup_mock, patch.object(
+            BackgroundJobService,
+            "mark_stale_active_jobs_failed",
+            return_value=3,
+        ) as jobs_cleanup_mock:
             result = parser_tasks.cleanup_stale_parser_loads(max_age_minutes=45)
 
         cleanup_mock.assert_called_once_with(max_age_minutes=45)
+        jobs_cleanup_mock.assert_called_once()
+        self.assertEqual(jobs_cleanup_mock.call_args.kwargs["max_age_minutes"], 45)
         self.assertEqual(result["status"], "success")
         self.assertEqual(result["marked_failed"], 2)
+        self.assertEqual(result["marked_jobs_failed"], 3)
         self.assertEqual(result["max_age_minutes"], 45)
 
+    def test_get_or_create_background_job_merges_meta_for_precreated_job(self):
+        BackgroundJobService.create_job(
+            task_id="precreated-task",
+            task_name="apps.parsers.tasks.parse_industrial_products",
+            user_id=None,
+            meta={
+                "source": ParserLoadLog.Source.INDUSTRIAL_PRODUCTS,
+                "source_card": "manufacturers-and-products",
+            },
+        )
+
+        job = parser_tasks._get_or_create_background_job(
+            task_id="precreated-task",
+            task_name="apps.parsers.tasks.parse_industrial_products",
+            source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS,
+            batch_id=7,
+            requested_by_id=42,
+            meta={"source_key": "mpt_products"},
+        )
+
+        self.assertEqual(job.user_id, 42)
+        self.assertEqual(job.meta["source_card"], "manufacturers-and-products")
+        self.assertEqual(job.meta["source_key"], "mpt_products")
+        self.assertEqual(job.meta["batch_id"], 7)
+
 
 @override_settings(
     CELERY_TASK_ALWAYS_EAGER=True,