372 lines
13 KiB
Python
372 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from apps.core.models import BackgroundJob, JobStatus
|
|
from apps.parsers.models import GenericParserRecord, ParserLoadLog
|
|
from apps.parsers.source_cards import (
|
|
SourceCardDefinition,
|
|
SourceCardService,
|
|
SourceItemDefinition,
|
|
)
|
|
from django.http import Http404
|
|
from django.test import SimpleTestCase, TestCase, override_settings
|
|
from django.utils import timezone
|
|
from rest_framework.exceptions import ValidationError
|
|
|
|
|
|
class SourceCardServiceUnitTest(SimpleTestCase):
|
|
def test_get_definition_raises_for_unknown_slug(self):
|
|
with self.assertRaises(Http404):
|
|
SourceCardService.get_definition("missing-card")
|
|
|
|
def test_validate_refresh_params_rejects_unknown_param(self):
|
|
definition = SourceCardService.get_definition("public-procurements")
|
|
|
|
with self.assertRaises(ValidationError) as error:
|
|
SourceCardService._validate_refresh_params(
|
|
definition,
|
|
{"region_code": "77", "unexpected": "value"},
|
|
)
|
|
|
|
self.assertIn("Неизвестные параметры обновления", str(error.exception.detail))
|
|
|
|
def test_validate_refresh_params_casts_integers(self):
|
|
definition = SourceCardService.get_definition("public-procurements")
|
|
|
|
validated = SourceCardService._validate_refresh_params(
|
|
definition,
|
|
{
|
|
"region_code": "77",
|
|
"current_year": "2025",
|
|
"current_month": "2",
|
|
},
|
|
)
|
|
|
|
self.assertEqual(
|
|
validated,
|
|
{
|
|
"region_code": "77",
|
|
"law_type": "44",
|
|
"current_year": 2025,
|
|
"current_month": 2,
|
|
},
|
|
)
|
|
|
|
def test_validate_refresh_params_raises_on_invalid_integer(self):
|
|
definition = SourceCardService.get_definition("public-procurements")
|
|
|
|
with self.assertRaises(ValidationError) as error:
|
|
SourceCardService._validate_refresh_params(
|
|
definition,
|
|
{"region_code": "77", "current_year": "not-a-number"},
|
|
)
|
|
|
|
self.assertIn("Значение должно быть целым числом", str(error.exception.detail))
|
|
|
|
@patch(
|
|
"apps.parsers.source_cards.SourceCardService._enqueue_task",
|
|
side_effect=[
|
|
{
|
|
"task_id": "task-1",
|
|
"task_name": "apps.parsers.tasks.parse_industrial_production",
|
|
},
|
|
{
|
|
"task_id": "task-2",
|
|
"task_name": "apps.parsers.tasks.parse_industrial_products",
|
|
},
|
|
{"task_id": "task-3", "task_name": "apps.parsers.tasks.parse_manufactures"},
|
|
],
|
|
)
|
|
def test_refresh_card_for_manufacturers_enqueues_three_tasks(self, enqueue_mock):
|
|
result = SourceCardService.refresh_card(
|
|
slug="manufacturers-and-products",
|
|
requested_by_id=12,
|
|
)
|
|
|
|
self.assertEqual(result["source_card"], "manufacturers-and-products")
|
|
self.assertEqual(
|
|
[item["task_id"] for item in result["tasks"]],
|
|
["task-1", "task-2", "task-3"],
|
|
)
|
|
self.assertEqual(enqueue_mock.call_count, 3)
|
|
|
|
@patch(
|
|
"apps.parsers.source_cards.SourceCardService._enqueue_task",
|
|
return_value={
|
|
"task_id": "task-1",
|
|
"task_name": "apps.parsers.tasks.sync_inspections",
|
|
},
|
|
)
|
|
def test_launch_refresh_for_inspections_passes_supported_kwargs_only(
|
|
self, enqueue_mock
|
|
):
|
|
definition = SourceCardService.get_definition("planned-inspections")
|
|
|
|
result = SourceCardService._launch_refresh(
|
|
definition,
|
|
requested_by_id=44,
|
|
params={
|
|
"current_year": 2025,
|
|
"current_month": 3,
|
|
"use_playwright": True,
|
|
"ignored": "value",
|
|
},
|
|
)
|
|
|
|
self.assertEqual(
|
|
result,
|
|
[{"task_id": "task-1", "task_name": "apps.parsers.tasks.sync_inspections"}],
|
|
)
|
|
self.assertEqual(
|
|
enqueue_mock.call_args.kwargs["kwargs"],
|
|
{
|
|
"requested_by_id": 44,
|
|
"current_year": 2025,
|
|
"current_month": 3,
|
|
"use_playwright": True,
|
|
},
|
|
)
|
|
|
|
@patch(
|
|
"apps.parsers.source_cards.SourceCardService._enqueue_task",
|
|
return_value={
|
|
"task_id": "task-9",
|
|
"task_name": "apps.parsers.tasks.sync_procurements",
|
|
},
|
|
)
|
|
def test_refresh_card_for_procurements_uses_default_law_type(self, enqueue_mock):
|
|
result = SourceCardService.refresh_card(
|
|
slug="public-procurements",
|
|
requested_by_id=10,
|
|
params={"region_code": "77", "current_year": "2026"},
|
|
)
|
|
|
|
self.assertEqual(result["source_card"], "public-procurements")
|
|
self.assertEqual(result["tasks"][0]["task_id"], "task-9")
|
|
self.assertEqual(
|
|
enqueue_mock.call_args.kwargs["kwargs"],
|
|
{
|
|
"requested_by_id": 10,
|
|
"region_code": "77",
|
|
"law_type": "44",
|
|
"current_year": 2026,
|
|
},
|
|
)
|
|
|
|
def test_launch_refresh_raises_for_unsupported_card(self):
|
|
definition = SourceCardDefinition(
|
|
slug="custom-source",
|
|
title="Custom",
|
|
description="Custom card",
|
|
order=999,
|
|
task_names=(),
|
|
source_items=(
|
|
SourceItemDefinition(
|
|
code="custom",
|
|
title="Custom",
|
|
description="Custom source",
|
|
),
|
|
),
|
|
)
|
|
|
|
with self.assertRaises(ValidationError) as error:
|
|
SourceCardService._launch_refresh(
|
|
definition,
|
|
requested_by_id=1,
|
|
params={},
|
|
)
|
|
|
|
self.assertIn(
|
|
"Обновление для карточки не поддерживается", str(error.exception.detail)
|
|
)
|
|
|
|
def test_enqueue_task_deletes_background_job_on_async_error(self):
|
|
task = MagicMock()
|
|
task.apply_async.side_effect = RuntimeError("broker down")
|
|
queryset = MagicMock()
|
|
|
|
with patch(
|
|
"apps.parsers.source_cards.uuid.uuid4", return_value="task-id-1"
|
|
), patch("apps.parsers.source_cards.BackgroundJobService.create_job"), patch(
|
|
"apps.parsers.source_cards.BackgroundJobService.get_queryset",
|
|
return_value=queryset,
|
|
), self.assertRaisesMessage(RuntimeError, "broker down"):
|
|
SourceCardService._enqueue_task(
|
|
task=task,
|
|
task_name="apps.parsers.tasks.sync_procurements",
|
|
requested_by_id=5,
|
|
meta={"source_card": "public-procurements"},
|
|
kwargs={"region_code": "77"},
|
|
)
|
|
|
|
queryset.filter.assert_called_once_with(task_id="task-id-1")
|
|
queryset.filter.return_value.delete.assert_called_once_with()
|
|
|
|
def test_helper_methods_cover_unknown_codes_and_status_variants(self):
|
|
self.assertEqual(SourceCardService._get_source_records_count("unknown"), 0)
|
|
self.assertEqual(
|
|
SourceCardService._get_source_organizations_count("unknown"), 0
|
|
)
|
|
self.assertIsNone(SourceCardService._get_source_data_timestamp("unknown"))
|
|
self.assertIsNone(SourceCardService._get_latest_load_by_source(None))
|
|
self.assertEqual(SourceCardService._get_status_label("custom"), "custom")
|
|
|
|
unavailable_definition = SourceCardDefinition(
|
|
slug="unavailable",
|
|
title="Unavailable",
|
|
description="Unavailable source",
|
|
order=1,
|
|
task_names=(),
|
|
source_items=(),
|
|
is_available=False,
|
|
)
|
|
in_progress_load = SimpleNamespace(status="in_progress")
|
|
failed_load = SimpleNamespace(status="failed")
|
|
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=unavailable_definition,
|
|
active_tasks=[],
|
|
latest_load=None,
|
|
last_updated_at=None,
|
|
),
|
|
"unavailable",
|
|
)
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=SourceCardService.get_definition("financial-indicators"),
|
|
active_tasks=[{"progress": 10}],
|
|
latest_load=None,
|
|
last_updated_at=None,
|
|
),
|
|
"in_progress",
|
|
)
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=SourceCardService.get_definition("financial-indicators"),
|
|
active_tasks=[],
|
|
latest_load=in_progress_load,
|
|
last_updated_at=None,
|
|
),
|
|
"in_progress",
|
|
)
|
|
stale_in_progress_load = SimpleNamespace(
|
|
status="in_progress",
|
|
updated_at=timezone.now() - timedelta(hours=3),
|
|
)
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=SourceCardService.get_definition("financial-indicators"),
|
|
active_tasks=[],
|
|
latest_load=stale_in_progress_load,
|
|
last_updated_at=None,
|
|
),
|
|
"error",
|
|
)
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=SourceCardService.get_definition("financial-indicators"),
|
|
active_tasks=[],
|
|
latest_load=failed_load,
|
|
last_updated_at=None,
|
|
),
|
|
"error",
|
|
)
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=SourceCardService.get_definition("financial-indicators"),
|
|
active_tasks=[],
|
|
latest_load=None,
|
|
last_updated_at=object(),
|
|
),
|
|
"success",
|
|
)
|
|
self.assertEqual(
|
|
SourceCardService._get_status(
|
|
definition=SourceCardService.get_definition("financial-indicators"),
|
|
active_tasks=[],
|
|
latest_load=None,
|
|
last_updated_at=None,
|
|
),
|
|
"idle",
|
|
)
|
|
|
|
|
|
@override_settings(PARSER_STALE_LOAD_MAX_AGE_MINUTES=90)
|
|
class SourceCardServiceDatabaseTest(TestCase):
|
|
def test_public_procurements_counts_generic_eis_sources(self):
|
|
GenericParserRecord.objects.create(
|
|
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
|
load_batch=1,
|
|
external_id="notice-1",
|
|
inn="7701234567",
|
|
title="Закупка 44-ФЗ",
|
|
payload={"number": "notice-1"},
|
|
)
|
|
GenericParserRecord.objects.create(
|
|
source=ParserLoadLog.Source.CONTRACTS,
|
|
load_batch=1,
|
|
external_id="contract-1",
|
|
inn="7701234567",
|
|
title="Контракт ЕИС",
|
|
payload={"number": "contract-1"},
|
|
)
|
|
ParserLoadLog.objects.create(
|
|
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
|
batch_id=1,
|
|
records_count=1,
|
|
status=ParserLoadLog.Status.SUCCESS,
|
|
)
|
|
ParserLoadLog.objects.create(
|
|
source=ParserLoadLog.Source.CONTRACTS,
|
|
batch_id=1,
|
|
records_count=1,
|
|
status=ParserLoadLog.Status.SUCCESS,
|
|
)
|
|
|
|
card = SourceCardService.get_card("public-procurements")
|
|
|
|
self.assertEqual(card["status"], "success")
|
|
self.assertEqual(card["records_count"], 2)
|
|
self.assertEqual(card["organizations_count"], 1)
|
|
|
|
def test_get_active_tasks_ignores_old_jobs_even_when_updated_recently(self):
|
|
job = BackgroundJob.objects.create(
|
|
task_id="old-source-task",
|
|
task_name="apps.parsers.tasks.parse_industrial_products",
|
|
status=JobStatus.STARTED,
|
|
progress=10,
|
|
meta={"source": "industrial_products"},
|
|
)
|
|
old_timestamp = timezone.now() - timedelta(hours=3)
|
|
BackgroundJob.objects.filter(pk=job.pk).update(
|
|
created_at=old_timestamp,
|
|
started_at=old_timestamp,
|
|
updated_at=timezone.now(),
|
|
)
|
|
|
|
tasks = SourceCardService._get_active_tasks(
|
|
SourceCardService.get_definition("manufacturers-and-products")
|
|
)
|
|
|
|
self.assertEqual(tasks, [])
|
|
|
|
def test_get_active_tasks_keeps_recent_pending_jobs(self):
|
|
BackgroundJob.objects.create(
|
|
task_id="fresh-source-task",
|
|
task_name="apps.parsers.tasks.parse_industrial_products",
|
|
status=JobStatus.PENDING,
|
|
progress=0,
|
|
meta={"source": "industrial_products"},
|
|
)
|
|
|
|
tasks = SourceCardService._get_active_tasks(
|
|
SourceCardService.get_definition("manufacturers-and-products")
|
|
)
|
|
|
|
self.assertEqual(len(tasks), 1)
|
|
self.assertEqual(tasks[0]["task_id"], "fresh-source-task")
|