Aleksandr Meshchriakov
052389d921
- перенесены тесты parsers из src/apps/parsers/tests в tests/apps/parsers - обновлены тесты задач под текущее поведение Celery (ошибки пробрасываются исключениями) - убрана зависимость тестов от внешнего брокера через локальные eager-вызовы - добавлены/уточнены фабрики и импорты для единой структуры тестов - обновлены README и CHANGELOG с новым правилом размещения тестов и запуском
344 lines
11 KiB
Python
344 lines
11 KiB
Python
"""
|
||
E2E тесты для парсера zakupki.gov.ru.
|
||
|
||
Тесты выполняются через локальный HTTP сервер (без внешних API).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from urllib.parse import urlparse
|
||
|
||
from apps.parsers.clients.zakupki import ZakupkiClient
|
||
from apps.parsers.models import ParserLoadLog, ProcurementRecord
|
||
from apps.parsers.services import ParserLoadLogService, ProcurementService
|
||
from django.test import TestCase, override_settings
|
||
|
||
from tests.utils import TestHTTPServer
|
||
from tests.utils.fixtures import build_zakupki_xml, build_zip, fake
|
||
|
||
|
||
def _digits(length: int) -> str:
|
||
return "".join(str(fake.random_int(0, 9)) for _ in range(length))
|
||
|
||
|
||
def _region_code() -> str:
|
||
return str(fake.random_int(min=1, max=99)).zfill(2)
|
||
|
||
|
||
def _host(server: TestHTTPServer) -> str:
|
||
parsed = urlparse(server.base_url)
|
||
if parsed.port:
|
||
return f"{parsed.hostname}:{parsed.port}"
|
||
return parsed.hostname or ""
|
||
|
||
|
||
def _add_zakupki_zip(
|
||
server: TestHTTPServer,
|
||
*,
|
||
region_code: str,
|
||
year: int,
|
||
month: int,
|
||
law_type: str = "44",
|
||
count: int = 3,
|
||
) -> int:
|
||
xml_bytes, rows = build_zakupki_xml(count=count)
|
||
zip_bytes = build_zip([(f"data_{region_code}_{year}_{month}.xml", xml_bytes)])
|
||
fz_suffix = f"fz{law_type}"
|
||
path = (
|
||
"/opendata/download/notifications/"
|
||
f"{region_code}/{year}/{month:02d}/{fz_suffix}.zip"
|
||
)
|
||
server.add_bytes(path, zip_bytes, content_type="application/zip")
|
||
return len(rows)
|
||
|
||
|
||
def _proxy_address() -> str:
|
||
return f"http://{fake.ipv4()}:{fake.port_number()}"
|
||
|
||
|
||
class ZakupkiClientE2ETestCase(TestCase):
|
||
"""
|
||
E2E тесты клиента ZakupkiClient через локальный HTTP сервер.
|
||
"""
|
||
|
||
def test_fetch_procurement_plans(self):
|
||
"""Получение списка файлов для загрузки."""
|
||
region_code = _region_code()
|
||
year = fake.random_int(min=2020, max=2026)
|
||
client = ZakupkiClient()
|
||
|
||
plans = client.fetch_procurement_plans(region_code=region_code, year=year)
|
||
|
||
self.assertIsInstance(plans, list)
|
||
self.assertGreater(len(plans), 0)
|
||
|
||
for plan in plans:
|
||
self.assertEqual(plan.region_code, region_code)
|
||
self.assertEqual(plan.year, year)
|
||
|
||
def test_fetch_procurements_with_missing_file(self):
|
||
"""Загрузка с отсутствующим файлом возвращает пустой список."""
|
||
region_code = _region_code()
|
||
year = fake.random_int(min=2020, max=2026)
|
||
month = fake.random_int(min=1, max=12)
|
||
|
||
with TestHTTPServer() as server:
|
||
client = ZakupkiClient(
|
||
host=_host(server),
|
||
scheme="http",
|
||
timeout=5,
|
||
http_adapter=server.adapter,
|
||
)
|
||
procurements = client.fetch_procurements(
|
||
region_code=region_code,
|
||
year=year,
|
||
month=month,
|
||
)
|
||
|
||
self.assertIsInstance(procurements, list)
|
||
self.assertEqual(procurements, [])
|
||
|
||
def test_fetch_with_progress_callback(self):
|
||
"""Тест callback для отслеживания прогресса."""
|
||
progress_updates: list[dict[str, object]] = []
|
||
region_code = _region_code()
|
||
year = fake.random_int(min=2020, max=2026)
|
||
month = fake.random_int(min=1, max=12)
|
||
|
||
def progress_callback(percent: int, message: str) -> None:
|
||
progress_updates.append({"percent": percent, "message": message})
|
||
|
||
with TestHTTPServer() as server:
|
||
_add_zakupki_zip(
|
||
server,
|
||
region_code=region_code,
|
||
year=year,
|
||
month=month,
|
||
)
|
||
client = ZakupkiClient(
|
||
host=_host(server),
|
||
scheme="http",
|
||
timeout=5,
|
||
http_adapter=server.adapter,
|
||
)
|
||
client.fetch_procurements(
|
||
region_code=region_code,
|
||
year=year,
|
||
month=month,
|
||
progress_callback=progress_callback,
|
||
)
|
||
|
||
self.assertGreater(len(progress_updates), 0)
|
||
self.assertEqual(progress_updates[0]["percent"], 0)
|
||
|
||
|
||
class ProcurementServiceE2ETestCase(TestCase):
|
||
"""
|
||
E2E тесты сервиса ProcurementService.
|
||
|
||
Тестирует полный цикл: загрузка → парсинг → сохранение в БД.
|
||
"""
|
||
|
||
def test_full_load_cycle(self):
|
||
"""Полный цикл загрузки данных."""
|
||
source = ParserLoadLog.Source.PROCUREMENTS
|
||
batch_id = ParserLoadLogService.get_next_batch_id(source)
|
||
region_code = _region_code()
|
||
year = fake.random_int(min=2020, max=2026)
|
||
month = fake.random_int(min=1, max=12)
|
||
|
||
load_log = ParserLoadLogService.create_load_log(
|
||
source=source,
|
||
batch_id=batch_id,
|
||
status="in_progress",
|
||
)
|
||
|
||
with TestHTTPServer() as server:
|
||
expected_count = _add_zakupki_zip(
|
||
server,
|
||
region_code=region_code,
|
||
year=year,
|
||
month=month,
|
||
)
|
||
with ZakupkiClient(
|
||
host=_host(server),
|
||
scheme="http",
|
||
timeout=5,
|
||
http_adapter=server.adapter,
|
||
) as client:
|
||
procurements = client.fetch_procurements(
|
||
region_code=region_code,
|
||
year=year,
|
||
month=month,
|
||
)
|
||
|
||
if procurements:
|
||
saved_count = ProcurementService.save_procurements(
|
||
procurements,
|
||
batch_id=batch_id,
|
||
region_code=region_code,
|
||
data_year=year,
|
||
data_month=month,
|
||
)
|
||
|
||
ParserLoadLogService.update(
|
||
load_log,
|
||
status="success",
|
||
records_count=saved_count,
|
||
)
|
||
|
||
self.assertGreater(saved_count, 0)
|
||
self.assertEqual(saved_count, expected_count)
|
||
self.assertEqual(ProcurementRecord.objects.count(), saved_count)
|
||
|
||
record = ProcurementRecord.objects.first()
|
||
self.assertIsNotNone(record.purchase_number)
|
||
self.assertEqual(record.region_code, region_code)
|
||
self.assertEqual(record.data_year, year)
|
||
self.assertEqual(record.data_month, month)
|
||
self.assertEqual(record.load_batch, batch_id)
|
||
|
||
load_log.refresh_from_db()
|
||
self.assertEqual(load_log.status, "success")
|
||
self.assertEqual(load_log.records_count, saved_count)
|
||
else:
|
||
ParserLoadLogService.update(
|
||
load_log,
|
||
status="success",
|
||
records_count=0,
|
||
)
|
||
|
||
|
||
@override_settings(
|
||
CELERY_TASK_ALWAYS_EAGER=True,
|
||
CELERY_TASK_EAGER_PROPAGATES=True,
|
||
)
|
||
class ProcurementTasksE2ETestCase(TestCase):
|
||
"""
|
||
E2E тесты Celery задач.
|
||
|
||
Запускает задачи в синхронном режиме с локальным HTTP сервером.
|
||
"""
|
||
|
||
def test_parse_procurements_task(self):
|
||
"""Тест задачи parse_procurements."""
|
||
import uuid
|
||
|
||
from apps.parsers.tasks import parse_procurements
|
||
|
||
region_code = _region_code()
|
||
year = fake.random_int(min=2020, max=2026)
|
||
month = fake.random_int(min=1, max=12)
|
||
|
||
with TestHTTPServer() as server:
|
||
_add_zakupki_zip(
|
||
server,
|
||
region_code=region_code,
|
||
year=year,
|
||
month=month,
|
||
)
|
||
task_id = str(uuid.uuid4())
|
||
async_result = parse_procurements.apply(
|
||
kwargs={
|
||
"region_code": region_code,
|
||
"year": year,
|
||
"month": month,
|
||
"law_type": "44",
|
||
"client_host": _host(server),
|
||
"client_scheme": "http",
|
||
"client_adapter": server.adapter,
|
||
},
|
||
task_id=task_id,
|
||
)
|
||
result = async_result.result
|
||
|
||
self.assertIn("batch_id", result)
|
||
self.assertIn("status", result)
|
||
self.assertIn("saved", result)
|
||
self.assertIn(result["status"], ["success", "failed"])
|
||
|
||
if result["status"] == "success":
|
||
self.assertGreater(result["batch_id"], 0)
|
||
|
||
|
||
class ZakupkiClientOfflineTestCase(TestCase):
|
||
"""
|
||
Тесты клиента без реальных HTTP запросов.
|
||
|
||
Проверяет обработку ошибок и edge cases.
|
||
"""
|
||
|
||
def test_client_handles_connection_error(self):
|
||
"""Клиент корректно обрабатывает ошибки соединения."""
|
||
client = ZakupkiClient(host="127.0.0.1:1", scheme="http", timeout=2)
|
||
|
||
procurements = client.fetch_procurements(
|
||
region_code=_region_code(),
|
||
year=fake.random_int(min=2020, max=2026),
|
||
file_url="http://127.0.0.1:1/data.xml",
|
||
)
|
||
|
||
self.assertEqual(procurements, [])
|
||
client.close()
|
||
|
||
def test_client_with_empty_response(self):
|
||
"""Клиент обрабатывает пустой ответ."""
|
||
client = ZakupkiClient()
|
||
|
||
procurements = client._parse_xml_content(
|
||
b'<?xml version="1.0"?><root></root>',
|
||
None,
|
||
)
|
||
|
||
self.assertEqual(procurements, [])
|
||
client.close()
|
||
|
||
|
||
class ProxyIntegrationTestCase(TestCase):
|
||
"""
|
||
Тесты интеграции с прокси.
|
||
|
||
Проверяет что клиент корректно использует прокси из БД.
|
||
"""
|
||
|
||
def test_client_accepts_proxy_list(self):
|
||
"""Клиент принимает список прокси."""
|
||
proxies = [_proxy_address(), _proxy_address()]
|
||
client = ZakupkiClient(proxies=proxies)
|
||
|
||
self.assertEqual(client.proxies, proxies)
|
||
client.close()
|
||
|
||
def test_client_works_without_proxies(self):
|
||
"""Клиент работает без прокси."""
|
||
client = ZakupkiClient()
|
||
|
||
self.assertIsNone(client.proxies)
|
||
client.close()
|
||
|
||
def test_proxy_service_integration(self):
|
||
"""Интеграция с ProxyService."""
|
||
from apps.parsers.services import ProxyService
|
||
|
||
from tests.apps.parsers.factories import ProxyFactory
|
||
|
||
ProxyFactory.create_batch(3, is_active=True)
|
||
ProxyFactory.create_batch(2, is_active=False)
|
||
|
||
proxies = ProxyService.get_active_proxies_or_none()
|
||
|
||
self.assertIsNotNone(proxies)
|
||
self.assertEqual(len(proxies), 3)
|
||
|
||
client = ZakupkiClient(proxies=proxies)
|
||
self.assertEqual(len(client.proxies), 3)
|
||
client.close()
|
||
|
||
def test_proxy_service_returns_none_when_empty(self):
|
||
"""ProxyService возвращает None если нет активных прокси."""
|
||
from apps.parsers.services import ProxyService
|
||
|
||
proxies = ProxyService.get_active_proxies_or_none()
|
||
|
||
self.assertIsNone(proxies)
|