Files
mostovik-backend/tests/apps/parsers/test_zakupki_client.py
Aleksandr Meshchriakov 052389d921 refactor(parsers): перенести тесты в ROOT_DIR/tests и синхронизировать контракты задач
- перенесены тесты parsers из src/apps/parsers/tests в tests/apps/parsers

- обновлены тесты задач под текущее поведение Celery (ошибки пробрасываются исключениями)

- убрана зависимость тестов от внешнего брокера через локальные eager-вызовы

- добавлены/уточнены фабрики и импорты для единой структуры тестов

- обновлены README и CHANGELOG с новым правилом размещения тестов и запуском
2026-03-04 15:35:50 +01:00

643 lines
23 KiB
Python

"""Unit tests for ZakupkiClient using local HTTP server (no mocks)."""
from __future__ import annotations
import io
import zipfile
from urllib.parse import urlparse
from apps.parsers.clients.zakupki import ZakupkiClient, ZakupkiClientError
from apps.parsers.clients.zakupki.schemas import Procurement, ProcurementPlan
from django.test import SimpleTestCase
from tests.utils import Response, TestHTTPServer
from tests.utils.fixtures import build_zakupki_xml, build_zip, fake
def _host_from_base_url(base_url: str) -> str:
parsed = urlparse(base_url)
if parsed.port:
return f"{parsed.hostname}:{parsed.port}"
return parsed.hostname or ""
def _proxy_address() -> str:
return f"http://{fake.ipv4()}:{fake.port_number()}"
def _digits(length: int) -> str:
return "".join(str(fake.random_int(0, 9)) for _ in range(length))
def _soap_response_with_archive(url: str) -> bytes:
xml = (
"<?xml version='1.0' encoding='utf-8'?>"
"<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope/'>"
"<soap:Body>"
"<ns2:getDocsByOrgRegionResponse xmlns:ns2='http://zakupki.gov.ru/fz44/get-docs-ip/ws'>"
"<dataInfo><archiveUrl>%s</archiveUrl></dataInfo>"
"</ns2:getDocsByOrgRegionResponse>"
"</soap:Body>"
"</soap:Envelope>"
) % url
return xml.encode("utf-8")
def _soap_response_with_error(code: str, message: str) -> bytes:
xml = (
( # noqa
"<?xml version='1.0' encoding='utf-8'?>"
"<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope/'>"
"<soap:Body>"
"<errorInfo><code>%s</code><message>%s</message></errorInfo>"
"</soap:Body>"
"</soap:Envelope>"
)
% (code, message)
)
return xml.encode("utf-8")
def _soap_response_with_fault(text: str) -> bytes:
xml = (
"<?xml version='1.0' encoding='utf-8'?>"
"<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope/'>"
"<soap:Body>"
"<soap:Fault>%s</soap:Fault>"
"</soap:Body>"
"</soap:Envelope>"
) % text
return xml.encode("utf-8")
class ZakupkiClientInitTestCase(SimpleTestCase):
def test_init_default(self):
client = ZakupkiClient()
self.assertEqual(client.host, "zakupki.gov.ru")
self.assertEqual(client.timeout, 120)
self.assertIsNone(client.proxies)
def test_init_with_proxies(self):
proxies = [_proxy_address(), _proxy_address()]
client = ZakupkiClient(proxies=proxies)
self.assertEqual(client.proxies, proxies)
def test_init_with_custom_timeout(self):
client = ZakupkiClient(timeout=60)
self.assertEqual(client.timeout, 60)
def test_context_manager(self):
with ZakupkiClient() as client:
self.assertIsInstance(client, ZakupkiClient)
class ZakupkiClientDiscoverFilesTestCase(SimpleTestCase):
def test_discover_files_with_region_and_year(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
client = ZakupkiClient()
plans = client._discover_data_files(region_code=region, year=year)
self.assertEqual(len(plans), 1)
self.assertIsInstance(plans[0], ProcurementPlan)
self.assertEqual(plans[0].region_code, region)
self.assertEqual(plans[0].year, year)
self.assertIsNone(plans[0].month)
def test_discover_files_with_month(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
month = fake.random_int(min=1, max=12)
client = ZakupkiClient()
plans = client._discover_data_files(region_code=region, year=year, month=month)
self.assertEqual(len(plans), 1)
self.assertEqual(plans[0].month, month)
self.assertIn(str(year), plans[0].file_url)
self.assertIn(f"{month:02d}", plans[0].file_url)
def test_discover_files_empty_without_region(self):
client = ZakupkiClient()
self.assertEqual(
client._discover_data_files(year=fake.random_int(min=2020, max=2026)), []
)
def test_discover_files_empty_without_year(self):
client = ZakupkiClient()
self.assertEqual(
client._discover_data_files(
region_code=f"{fake.random_int(min=1, max=99):02d}"
),
[],
)
def test_discover_files_law_type_44(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
client = ZakupkiClient()
plans = client._discover_data_files(
region_code=region, year=year, law_type="44"
)
self.assertIn("fz44", plans[0].file_url)
def test_discover_files_law_type_223(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
client = ZakupkiClient()
plans = client._discover_data_files(
region_code=region, year=year, law_type="223"
)
self.assertIn("fz223", plans[0].file_url)
class ZakupkiClientParseXMLTestCase(SimpleTestCase):
def setUp(self):
self.client = ZakupkiClient()
self.valid_xml, self.rows = build_zakupki_xml(count=2)
self.empty_xml = b"<?xml version='1.0' encoding='UTF-8'?><export></export>"
self.invalid_xml = b"not xml content"
def test_parse_xml_valid(self):
records = self.client._parse_xml_content(self.valid_xml)
self.assertEqual(len(records), len(self.rows))
self.assertIsInstance(records[0], Procurement)
def test_parse_xml_empty(self):
records = self.client._parse_xml_content(self.empty_xml)
self.assertEqual(records, [])
def test_parse_xml_invalid(self):
with self.assertRaises(ZakupkiClientError):
self.client._parse_xml_content(self.invalid_xml)
class ZakupkiClientParseZIPTestCase(SimpleTestCase):
def test_parse_zip_with_xml(self):
xml_content, rows = build_zakupki_xml(count=2)
archive = build_zip([("data.xml", xml_content)])
client = ZakupkiClient()
records = client._parse_zip_archive(archive)
self.assertEqual(len(records), len(rows))
def test_parse_zip_empty(self):
buf = io.BytesIO()
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED):
pass
client = ZakupkiClient()
records = client._parse_zip_archive(buf.getvalue())
self.assertEqual(records, [])
def test_parse_zip_multiple_xml_files(self):
xml_a, rows_a = build_zakupki_xml(count=1)
xml_b, rows_b = build_zakupki_xml(count=1)
archive = build_zip([("a.xml", xml_a), ("b.xml", xml_b)])
client = ZakupkiClient()
records = client._parse_zip_archive(archive)
self.assertEqual(len(records), len(rows_a) + len(rows_b))
class ZakupkiClientFetchTestCase(SimpleTestCase):
def test_fetch_with_region_and_year(self):
xml_content, rows = build_zakupki_xml(count=2)
archive = build_zip([("data.xml", xml_content)])
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
month = fake.random_int(min=1, max=12)
with TestHTTPServer() as server:
file_url = (
f"/opendata/download/notifications/{region}/{year}/{month:02d}/fz44.zip"
)
server.add_bytes(file_url, archive, content_type="application/zip")
client = ZakupkiClient(
host=_host_from_base_url(server.base_url),
scheme="http",
http_adapter=server.adapter,
)
procurements = client.fetch_procurements(
region_code=region,
year=year,
month=month,
law_type="44",
)
self.assertEqual(len(procurements), len(rows))
def test_fetch_with_direct_url(self):
xml_content, rows = build_zakupki_xml(count=2)
archive = build_zip([("data.xml", xml_content)])
with TestHTTPServer() as server:
server.add_bytes("/files/data.zip", archive, content_type="application/zip")
client = ZakupkiClient(
host=_host_from_base_url(server.base_url),
scheme="http",
http_adapter=server.adapter,
)
procurements = client.fetch_procurements(
file_url=f"{server.base_url}/files/data.zip"
)
self.assertEqual(len(procurements), len(rows))
def test_fetch_no_files_found(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
with TestHTTPServer() as server:
client = ZakupkiClient(
host=_host_from_base_url(server.base_url),
scheme="http",
http_adapter=server.adapter,
)
procurements = client.fetch_procurements(
region_code=region,
year=year,
law_type="44",
)
self.assertEqual(procurements, [])
def test_fetch_progress_callback(self):
xml_content, rows = build_zakupki_xml(count=1)
archive = build_zip([("data.xml", xml_content)])
progress = []
def progress_callback(percent: int, message: str) -> None:
progress.append((percent, message))
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
with TestHTTPServer() as server:
file_url = f"/opendata/download/notifications/{region}/{year}/fz44.zip"
server.add_bytes(file_url, archive, content_type="application/zip")
client = ZakupkiClient(
host=_host_from_base_url(server.base_url),
scheme="http",
http_adapter=server.adapter,
)
procurements = client.fetch_procurements(
region_code=region,
year=year,
law_type="44",
progress_callback=progress_callback,
)
self.assertEqual(len(procurements), len(rows))
self.assertTrue(progress)
def test_fetch_procurements_wraps_unexpected_error(self):
class _FailClient(ZakupkiClient):
def _fetch_via_http(self, **_kwargs): # type: ignore[override]
raise ValueError("boom")
client = _FailClient()
with self.assertRaises(ZakupkiClientError):
client.fetch_procurements(
region_code=f"{fake.random_int(min=1, max=99):02d}",
year=fake.random_int(min=2020, max=2025),
law_type="44",
)
class ZakupkiClientSanitizeXMLTestCase(SimpleTestCase):
def test_sanitize_removes_control_chars(self):
client = ZakupkiClient()
xml = "<a>\x01\x02</a>"
sanitized = client._sanitize_xml(xml)
self.assertEqual(sanitized, "<a></a>")
def test_sanitize_escapes_ampersands(self):
client = ZakupkiClient()
xml = "<a>Tom & Jerry</a>"
sanitized = client._sanitize_xml(xml)
self.assertIn("Tom &amp; Jerry", sanitized)
def test_sanitize_keeps_valid_entities(self):
client = ZakupkiClient()
xml = "<a>Tom &amp; Jerry</a>"
sanitized = client._sanitize_xml(xml)
self.assertEqual(sanitized, xml)
class ZakupkiClientSoapTestCase(SimpleTestCase):
def test_parse_soap_response_archive(self):
client = ZakupkiClient(token="token") # noqa
url = f"https://{fake.domain_name()}/archive.zip"
result = client._parse_soap_response(_soap_response_with_archive(url))
self.assertEqual(result, url)
def test_parse_soap_response_error(self):
client = ZakupkiClient(token="token") # noqa
with self.assertRaises(ZakupkiClientError):
client._parse_soap_response(_soap_response_with_error("400", "bad"))
def test_parse_soap_response_fault(self):
client = ZakupkiClient(token="token") # noqa
with self.assertRaises(ZakupkiClientError):
client._parse_soap_response(_soap_response_with_fault("fault"))
def test_parse_soap_response_invalid_xml(self):
client = ZakupkiClient(token="token") # noqa
with self.assertRaises(ZakupkiClientError):
client._parse_soap_response(b"<xml")
def test_parse_soap_response_without_archive(self):
client = ZakupkiClient(token="token") # noqa
xml = (
"<?xml version='1.0' encoding='utf-8'?>"
"<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope/'>"
"<soap:Body><dataInfo></dataInfo></soap:Body></soap:Envelope>"
)
result = client._parse_soap_response(xml.encode("utf-8"))
self.assertIsNone(result)
def test_fetch_via_soap_success(self):
xml_content, rows = build_zakupki_xml(count=1)
archive = build_zip([("data.xml", xml_content)])
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
with TestHTTPServer() as server:
archive_url = f"{server.base_url}/archive.zip"
server.add_route(
"POST",
"/soap",
lambda _req, _body: Response(
status=200,
body=_soap_response_with_archive(archive_url),
headers={"Content-Type": "text/xml"},
),
)
server.add_bytes("/archive.zip", archive, content_type="application/zip")
client = ZakupkiClient(
token="token", # noqa
host=_host_from_base_url(server.base_url),
scheme="http",
soap_url=f"{server.base_url}/soap",
http_adapter=server.adapter,
)
procurements = client.fetch_procurements(
region_code=region,
year=year,
law_type="44",
)
self.assertEqual(len(procurements), len(rows))
def test_fetch_via_soap_request_error(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
with TestHTTPServer() as server:
server.add_route(
"POST",
"/soap",
lambda _req, _body: Response(
status=500,
body=fake.pystr(min_chars=5, max_chars=10).encode("utf-8"),
headers={"Content-Type": "text/xml"},
),
)
client = ZakupkiClient(
token="token", # noqa
host=_host_from_base_url(server.base_url),
scheme="http",
soap_url=f"{server.base_url}/soap",
http_adapter=server.adapter,
)
with self.assertRaises(ZakupkiClientError):
client.fetch_procurements(region_code=region, year=year)
def test_fetch_via_soap_success_with_progress(self):
xml_content, rows = build_zakupki_xml(count=1)
archive = build_zip([("data.xml", xml_content)])
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
progress: list[int] = []
def on_progress(value: int, _message: str) -> None:
progress.append(value)
with TestHTTPServer() as server:
archive_url = f"{server.base_url}/archive.zip"
server.add_route(
"POST",
"/soap",
lambda _req, _body: Response(
status=200,
body=_soap_response_with_archive(archive_url),
headers={"Content-Type": "text/xml"},
),
)
server.add_bytes("/archive.zip", archive, content_type="application/zip")
client = ZakupkiClient(
token="token", # noqa
host=_host_from_base_url(server.base_url),
scheme="http",
soap_url=f"{server.base_url}/soap",
http_adapter=server.adapter,
)
procurements = client.fetch_procurements(
region_code=region,
year=year,
law_type="44",
progress_callback=on_progress,
)
self.assertEqual(len(procurements), len(rows))
self.assertTrue(progress)
def test_fetch_via_soap_requires_token(self):
client = ZakupkiClient(token=None)
with self.assertRaises(ZakupkiClientError):
client._fetch_via_soap(region_code="77", year=2025)
def test_fetch_via_soap_requires_params(self):
client = ZakupkiClient(token="token") # noqa
with self.assertRaises(ZakupkiClientError):
client._fetch_via_soap()
def test_fetch_via_soap_no_archive_url(self):
progress = []
def on_progress(value: int, _message: str) -> None:
progress.append(value)
with TestHTTPServer() as server:
server.add_route(
"POST",
"/soap",
lambda _req, _body: Response(
status=200,
body=(
b"<?xml version='1.0' encoding='utf-8'?>"
b"<soap:Envelope xmlns:soap='http://schemas.xmlsoap.org/soap/envelope/'>"
b"<soap:Body><dataInfo></dataInfo></soap:Body></soap:Envelope>"
),
headers={"Content-Type": "text/xml"},
),
)
client = ZakupkiClient(
token="token", # noqa
host=_host_from_base_url(server.base_url),
scheme="http",
soap_url=f"{server.base_url}/soap",
http_adapter=server.adapter,
)
result = client._fetch_via_soap(
region_code="77", year=2025, progress_callback=on_progress
)
self.assertEqual(result, [])
self.assertTrue(progress)
def test_fetch_by_reestr_number(self):
xml_content, rows = build_zakupki_xml(count=1)
archive = build_zip([("data.xml", xml_content)])
reestr = _digits(19)
with TestHTTPServer() as server:
archive_url = f"{server.base_url}/archive.zip"
server.add_route(
"POST",
"/soap",
lambda _req, _body: Response(
status=200,
body=_soap_response_with_archive(archive_url),
headers={"Content-Type": "text/xml"},
),
)
server.add_bytes("/archive.zip", archive, content_type="application/zip")
client = ZakupkiClient(
token="token", # noqa
host=_host_from_base_url(server.base_url),
scheme="http",
soap_url=f"{server.base_url}/soap",
http_adapter=server.adapter,
)
procurements = client.fetch_by_reestr_number(reestr)
self.assertEqual(len(procurements), len(rows))
def test_fetch_via_soap_download_error(self):
region = f"{fake.random_int(min=1, max=99):02d}"
year = fake.random_int(min=2020, max=2025)
with TestHTTPServer() as server:
archive_url = f"{server.base_url}/archive.zip"
server.add_route(
"POST",
"/soap",
lambda _req, _body: Response(
status=200,
body=_soap_response_with_archive(archive_url),
headers={"Content-Type": "text/xml"},
),
)
server.add_bytes("/archive.zip", b"", status=500)
client = ZakupkiClient(
token="token", # noqa
host=_host_from_base_url(server.base_url),
scheme="http",
soap_url=f"{server.base_url}/soap",
http_adapter=server.adapter,
)
with self.assertRaises(ZakupkiClientError):
client._fetch_via_soap(region_code=region, year=year)
def test_build_soap_request_by_region_dates(self):
client = ZakupkiClient(token="token") # noqa
with_day = client._build_soap_request_by_region(
region_code="77", year=2024, month=1, day=2
)
self.assertIn("<exactDate>2024-01-02</exactDate>", with_day)
with_month = client._build_soap_request_by_region(
region_code="77", year=2024, month=3
)
self.assertIn("<exactDate>2024-03-01</exactDate>", with_month)
with_year = client._build_soap_request_by_region(region_code="77", year=2024)
self.assertIn("<exactDate>2024-01-01</exactDate>", with_year)
no_date = client._build_soap_request_by_region(region_code="77")
self.assertIn("exactDate", no_date)
def test_download_and_parse_http_xml(self):
xml_content, rows = build_zakupki_xml(count=1)
with TestHTTPServer() as server:
server.add_bytes(
"/files/data.xml",
xml_content,
content_type="application/xml",
)
client = ZakupkiClient(
host=_host_from_base_url(server.base_url),
scheme="http",
http_adapter=server.adapter,
)
procurements = client._download_and_parse_http(
f"{server.base_url}/files/data.xml"
)
self.assertEqual(len(procurements), len(rows))
class ZakupkiClientAdditionalParsingTestCase(SimpleTestCase):
def test_parse_zip_with_progress(self):
xml_content, rows = build_zakupki_xml(count=1)
archive = build_zip([("data.xml", xml_content)])
progress = []
def on_progress(value: int, _message: str) -> None:
progress.append(value)
client = ZakupkiClient()
records = client._parse_zip_archive(archive, progress_callback=on_progress)
self.assertEqual(len(records), len(rows))
def test_fetch_via_http_without_plans(self):
client = ZakupkiClient()
result = client._fetch_via_http(
region_code=None,
year=None,
month=None,
law_type="44",
)
self.assertEqual(result, [])
def test_parse_xml_with_namespace_and_encoding(self):
ns = "http://example.com"
xml = (
"<?xml version='1.0' encoding='windows-1251'?>"
f"<ns:export xmlns:ns='{ns}'>"
"<ns:notification>"
"<ns:purchaseNumber>123</ns:purchaseNumber>"
"<ns:purchaseName>Test</ns:purchaseName>"
"</ns:notification>"
"</ns:export>"
)
content = xml.encode("cp1251")
client = ZakupkiClient()
records = client._parse_xml_content(content)
self.assertEqual(len(records), 1)
def test_parse_xml_record_from_attributes(self):
xml = (
"<?xml version='1.0' encoding='utf-8'?>"
"<export>"
"<record purchaseNumber='321' purchaseName='AttrName' customerName='C'/>"
"</export>"
)
client = ZakupkiClient()
records = client._parse_xml_content(xml.encode("utf-8"))
self.assertEqual(len(records), 1)