fix(parsers): make source jobs resumable
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 1m11s
CI/CD Pipeline / Build and Push Images (push) Successful in 19s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 24s

This commit is contained in:
2026-08-14 17:02:56 +02:00
parent a0eb0a5303
commit 31eaa7e907
14 changed files with 884 additions and 476 deletions

View File

@@ -2,9 +2,11 @@ from __future__ import annotations
from datetime import datetime, timedelta
from io import BytesIO
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import patch
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.gosedo import (
GosedoNotModified,
GosedoParseResult,
@@ -22,6 +24,9 @@ from apps.parsers.media_news import (
)
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
from apps.parsers.source_artifacts import cleanup_parser_source_artifacts
from apps.parsers.tasks import parse_media_news
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase
from django.urls import reverse
@@ -519,6 +524,55 @@ class MediaNewsImportTest(TestCase):
)
class MediaNewsTaskTest(TestCase):
def test_uploaded_workbook_task_completes_background_job(self):
Organization.objects.create(
name="АО Задача СМИ",
inn="0012345678",
ogrn="1027700132195",
okpo="00123456",
directory_imported_at=timezone.now(),
)
workbook = _media_workbook(
[
[
"00123456",
"0012345678",
"2026-07-01",
"СМИ",
"https://example.test/news",
"Источник\nЗаголовок\nЛид\nСтрока 4\nПолный текст",
"Положительная",
]
]
)
with TemporaryDirectory() as media_root, self.settings(MEDIA_ROOT=media_root):
file_path = default_storage.save(
"parser_uploads/media.xlsx",
ContentFile(workbook.getvalue()),
)
task_result = parse_media_news.apply(
kwargs={
"file_path": file_path,
"original_name": "media.xlsx",
},
task_id="media-news-task-test",
)
self.assertTrue(task_result.successful())
self.assertFalse(default_storage.exists(file_path))
job = BackgroundJob.objects.get(task_id="media-news-task-test")
self.assertEqual(job.status, JobStatus.SUCCESS)
self.assertEqual(job.progress, 100)
self.assertEqual(task_result.result["published"], 1)
self.assertEqual(
ParserLoadLog.objects.get(source=ParserLoadLog.Source.MEDIA_NEWS).status,
ParserLoadLog.Status.SUCCESS,
)
class MediaNewsPermissionsTest(APITestCase):
def setUp(self):
self.user = UserFactory.create_user()
@@ -536,8 +590,11 @@ class MediaNewsPermissionsTest(APITestCase):
self.client.force_authenticate(self.admin)
with patch(
"apps.parsers.views._save_uploaded_parser_file",
return_value="parser_uploads/media.xlsx",
), patch(
"apps.parsers.tasks.parse_media_news.apply_async",
return_value=SimpleNamespace(id="media-task-1"),
side_effect=lambda **kwargs: SimpleNamespace(id=kwargs["task_id"]),
):
response = self.client.post(
self.url,
@@ -546,8 +603,21 @@ class MediaNewsPermissionsTest(APITestCase):
)
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
self.assertEqual(response.data["data"]["task_id"], "media-task-1")
self.assertEqual(response.data["data"]["task_ids"], ["media-task-1"])
task_id = response.data["data"]["task_id"]
self.assertEqual(response.data["data"]["task_ids"], [task_id])
card_response = self.client.get(
reverse(
"api_v1:sources:source-cards-detail",
kwargs={"slug": "media-mentions"},
)
)
self.assertEqual(card_response.status_code, status.HTTP_200_OK)
self.assertEqual(card_response.data["data"]["status"], "in_progress")
self.assertEqual(
card_response.data["data"]["active_tasks"][0]["task_id"],
task_id,
)
def test_gosedo_manual_run_is_admin_only_and_returns_task_ids(self):
url = reverse(