refactor(parsers): перенести тесты в ROOT_DIR/tests и синхронизировать контракты задач
- перенесены тесты parsers из src/apps/parsers/tests в tests/apps/parsers - обновлены тесты задач под текущее поведение Celery (ошибки пробрасываются исключениями) - убрана зависимость тестов от внешнего брокера через локальные eager-вызовы - добавлены/уточнены фабрики и импорты для единой структуры тестов - обновлены README и CHANGELOG с новым правилом размещения тестов и запуском
This commit is contained in:
343
tests/apps/parsers/test_e2e.py
Normal file
343
tests/apps/parsers/test_e2e.py
Normal file
@@ -0,0 +1,343 @@
|
||||
"""
|
||||
E2E тесты для парсера zakupki.gov.ru.
|
||||
|
||||
Тесты выполняются через локальный HTTP сервер (без внешних API).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from apps.parsers.clients.zakupki import ZakupkiClient
|
||||
from apps.parsers.models import ParserLoadLog, ProcurementRecord
|
||||
from apps.parsers.services import ParserLoadLogService, ProcurementService
|
||||
from django.test import TestCase, override_settings
|
||||
|
||||
from tests.utils import TestHTTPServer
|
||||
from tests.utils.fixtures import build_zakupki_xml, build_zip, fake
|
||||
|
||||
|
||||
def _digits(length: int) -> str:
|
||||
return "".join(str(fake.random_int(0, 9)) for _ in range(length))
|
||||
|
||||
|
||||
def _region_code() -> str:
|
||||
return str(fake.random_int(min=1, max=99)).zfill(2)
|
||||
|
||||
|
||||
def _host(server: TestHTTPServer) -> str:
|
||||
parsed = urlparse(server.base_url)
|
||||
if parsed.port:
|
||||
return f"{parsed.hostname}:{parsed.port}"
|
||||
return parsed.hostname or ""
|
||||
|
||||
|
||||
def _add_zakupki_zip(
|
||||
server: TestHTTPServer,
|
||||
*,
|
||||
region_code: str,
|
||||
year: int,
|
||||
month: int,
|
||||
law_type: str = "44",
|
||||
count: int = 3,
|
||||
) -> int:
|
||||
xml_bytes, rows = build_zakupki_xml(count=count)
|
||||
zip_bytes = build_zip([(f"data_{region_code}_{year}_{month}.xml", xml_bytes)])
|
||||
fz_suffix = f"fz{law_type}"
|
||||
path = (
|
||||
"/opendata/download/notifications/"
|
||||
f"{region_code}/{year}/{month:02d}/{fz_suffix}.zip"
|
||||
)
|
||||
server.add_bytes(path, zip_bytes, content_type="application/zip")
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _proxy_address() -> str:
|
||||
return f"http://{fake.ipv4()}:{fake.port_number()}"
|
||||
|
||||
|
||||
class ZakupkiClientE2ETestCase(TestCase):
|
||||
"""
|
||||
E2E тесты клиента ZakupkiClient через локальный HTTP сервер.
|
||||
"""
|
||||
|
||||
def test_fetch_procurement_plans(self):
|
||||
"""Получение списка файлов для загрузки."""
|
||||
region_code = _region_code()
|
||||
year = fake.random_int(min=2020, max=2026)
|
||||
client = ZakupkiClient()
|
||||
|
||||
plans = client.fetch_procurement_plans(region_code=region_code, year=year)
|
||||
|
||||
self.assertIsInstance(plans, list)
|
||||
self.assertGreater(len(plans), 0)
|
||||
|
||||
for plan in plans:
|
||||
self.assertEqual(plan.region_code, region_code)
|
||||
self.assertEqual(plan.year, year)
|
||||
|
||||
def test_fetch_procurements_with_missing_file(self):
|
||||
"""Загрузка с отсутствующим файлом возвращает пустой список."""
|
||||
region_code = _region_code()
|
||||
year = fake.random_int(min=2020, max=2026)
|
||||
month = fake.random_int(min=1, max=12)
|
||||
|
||||
with TestHTTPServer() as server:
|
||||
client = ZakupkiClient(
|
||||
host=_host(server),
|
||||
scheme="http",
|
||||
timeout=5,
|
||||
http_adapter=server.adapter,
|
||||
)
|
||||
procurements = client.fetch_procurements(
|
||||
region_code=region_code,
|
||||
year=year,
|
||||
month=month,
|
||||
)
|
||||
|
||||
self.assertIsInstance(procurements, list)
|
||||
self.assertEqual(procurements, [])
|
||||
|
||||
def test_fetch_with_progress_callback(self):
|
||||
"""Тест callback для отслеживания прогресса."""
|
||||
progress_updates: list[dict[str, object]] = []
|
||||
region_code = _region_code()
|
||||
year = fake.random_int(min=2020, max=2026)
|
||||
month = fake.random_int(min=1, max=12)
|
||||
|
||||
def progress_callback(percent: int, message: str) -> None:
|
||||
progress_updates.append({"percent": percent, "message": message})
|
||||
|
||||
with TestHTTPServer() as server:
|
||||
_add_zakupki_zip(
|
||||
server,
|
||||
region_code=region_code,
|
||||
year=year,
|
||||
month=month,
|
||||
)
|
||||
client = ZakupkiClient(
|
||||
host=_host(server),
|
||||
scheme="http",
|
||||
timeout=5,
|
||||
http_adapter=server.adapter,
|
||||
)
|
||||
client.fetch_procurements(
|
||||
region_code=region_code,
|
||||
year=year,
|
||||
month=month,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
self.assertGreater(len(progress_updates), 0)
|
||||
self.assertEqual(progress_updates[0]["percent"], 0)
|
||||
|
||||
|
||||
class ProcurementServiceE2ETestCase(TestCase):
|
||||
"""
|
||||
E2E тесты сервиса ProcurementService.
|
||||
|
||||
Тестирует полный цикл: загрузка → парсинг → сохранение в БД.
|
||||
"""
|
||||
|
||||
def test_full_load_cycle(self):
|
||||
"""Полный цикл загрузки данных."""
|
||||
source = ParserLoadLog.Source.PROCUREMENTS
|
||||
batch_id = ParserLoadLogService.get_next_batch_id(source)
|
||||
region_code = _region_code()
|
||||
year = fake.random_int(min=2020, max=2026)
|
||||
month = fake.random_int(min=1, max=12)
|
||||
|
||||
load_log = ParserLoadLogService.create_load_log(
|
||||
source=source,
|
||||
batch_id=batch_id,
|
||||
status="in_progress",
|
||||
)
|
||||
|
||||
with TestHTTPServer() as server:
|
||||
expected_count = _add_zakupki_zip(
|
||||
server,
|
||||
region_code=region_code,
|
||||
year=year,
|
||||
month=month,
|
||||
)
|
||||
with ZakupkiClient(
|
||||
host=_host(server),
|
||||
scheme="http",
|
||||
timeout=5,
|
||||
http_adapter=server.adapter,
|
||||
) as client:
|
||||
procurements = client.fetch_procurements(
|
||||
region_code=region_code,
|
||||
year=year,
|
||||
month=month,
|
||||
)
|
||||
|
||||
if procurements:
|
||||
saved_count = ProcurementService.save_procurements(
|
||||
procurements,
|
||||
batch_id=batch_id,
|
||||
region_code=region_code,
|
||||
data_year=year,
|
||||
data_month=month,
|
||||
)
|
||||
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status="success",
|
||||
records_count=saved_count,
|
||||
)
|
||||
|
||||
self.assertGreater(saved_count, 0)
|
||||
self.assertEqual(saved_count, expected_count)
|
||||
self.assertEqual(ProcurementRecord.objects.count(), saved_count)
|
||||
|
||||
record = ProcurementRecord.objects.first()
|
||||
self.assertIsNotNone(record.purchase_number)
|
||||
self.assertEqual(record.region_code, region_code)
|
||||
self.assertEqual(record.data_year, year)
|
||||
self.assertEqual(record.data_month, month)
|
||||
self.assertEqual(record.load_batch, batch_id)
|
||||
|
||||
load_log.refresh_from_db()
|
||||
self.assertEqual(load_log.status, "success")
|
||||
self.assertEqual(load_log.records_count, saved_count)
|
||||
else:
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status="success",
|
||||
records_count=0,
|
||||
)
|
||||
|
||||
|
||||
@override_settings(
|
||||
CELERY_TASK_ALWAYS_EAGER=True,
|
||||
CELERY_TASK_EAGER_PROPAGATES=True,
|
||||
)
|
||||
class ProcurementTasksE2ETestCase(TestCase):
|
||||
"""
|
||||
E2E тесты Celery задач.
|
||||
|
||||
Запускает задачи в синхронном режиме с локальным HTTP сервером.
|
||||
"""
|
||||
|
||||
def test_parse_procurements_task(self):
|
||||
"""Тест задачи parse_procurements."""
|
||||
import uuid
|
||||
|
||||
from apps.parsers.tasks import parse_procurements
|
||||
|
||||
region_code = _region_code()
|
||||
year = fake.random_int(min=2020, max=2026)
|
||||
month = fake.random_int(min=1, max=12)
|
||||
|
||||
with TestHTTPServer() as server:
|
||||
_add_zakupki_zip(
|
||||
server,
|
||||
region_code=region_code,
|
||||
year=year,
|
||||
month=month,
|
||||
)
|
||||
task_id = str(uuid.uuid4())
|
||||
async_result = parse_procurements.apply(
|
||||
kwargs={
|
||||
"region_code": region_code,
|
||||
"year": year,
|
||||
"month": month,
|
||||
"law_type": "44",
|
||||
"client_host": _host(server),
|
||||
"client_scheme": "http",
|
||||
"client_adapter": server.adapter,
|
||||
},
|
||||
task_id=task_id,
|
||||
)
|
||||
result = async_result.result
|
||||
|
||||
self.assertIn("batch_id", result)
|
||||
self.assertIn("status", result)
|
||||
self.assertIn("saved", result)
|
||||
self.assertIn(result["status"], ["success", "failed"])
|
||||
|
||||
if result["status"] == "success":
|
||||
self.assertGreater(result["batch_id"], 0)
|
||||
|
||||
|
||||
class ZakupkiClientOfflineTestCase(TestCase):
|
||||
"""
|
||||
Тесты клиента без реальных HTTP запросов.
|
||||
|
||||
Проверяет обработку ошибок и edge cases.
|
||||
"""
|
||||
|
||||
def test_client_handles_connection_error(self):
|
||||
"""Клиент корректно обрабатывает ошибки соединения."""
|
||||
client = ZakupkiClient(host="127.0.0.1:1", scheme="http", timeout=2)
|
||||
|
||||
procurements = client.fetch_procurements(
|
||||
region_code=_region_code(),
|
||||
year=fake.random_int(min=2020, max=2026),
|
||||
file_url="http://127.0.0.1:1/data.xml",
|
||||
)
|
||||
|
||||
self.assertEqual(procurements, [])
|
||||
client.close()
|
||||
|
||||
def test_client_with_empty_response(self):
|
||||
"""Клиент обрабатывает пустой ответ."""
|
||||
client = ZakupkiClient()
|
||||
|
||||
procurements = client._parse_xml_content(
|
||||
b'<?xml version="1.0"?><root></root>',
|
||||
None,
|
||||
)
|
||||
|
||||
self.assertEqual(procurements, [])
|
||||
client.close()
|
||||
|
||||
|
||||
class ProxyIntegrationTestCase(TestCase):
|
||||
"""
|
||||
Тесты интеграции с прокси.
|
||||
|
||||
Проверяет что клиент корректно использует прокси из БД.
|
||||
"""
|
||||
|
||||
def test_client_accepts_proxy_list(self):
|
||||
"""Клиент принимает список прокси."""
|
||||
proxies = [_proxy_address(), _proxy_address()]
|
||||
client = ZakupkiClient(proxies=proxies)
|
||||
|
||||
self.assertEqual(client.proxies, proxies)
|
||||
client.close()
|
||||
|
||||
def test_client_works_without_proxies(self):
|
||||
"""Клиент работает без прокси."""
|
||||
client = ZakupkiClient()
|
||||
|
||||
self.assertIsNone(client.proxies)
|
||||
client.close()
|
||||
|
||||
def test_proxy_service_integration(self):
|
||||
"""Интеграция с ProxyService."""
|
||||
from apps.parsers.services import ProxyService
|
||||
|
||||
from tests.apps.parsers.factories import ProxyFactory
|
||||
|
||||
ProxyFactory.create_batch(3, is_active=True)
|
||||
ProxyFactory.create_batch(2, is_active=False)
|
||||
|
||||
proxies = ProxyService.get_active_proxies_or_none()
|
||||
|
||||
self.assertIsNotNone(proxies)
|
||||
self.assertEqual(len(proxies), 3)
|
||||
|
||||
client = ZakupkiClient(proxies=proxies)
|
||||
self.assertEqual(len(client.proxies), 3)
|
||||
client.close()
|
||||
|
||||
def test_proxy_service_returns_none_when_empty(self):
|
||||
"""ProxyService возвращает None если нет активных прокси."""
|
||||
from apps.parsers.services import ProxyService
|
||||
|
||||
proxies = ProxyService.get_active_proxies_or_none()
|
||||
|
||||
self.assertIsNone(proxies)
|
||||
Reference in New Issue
Block a user