fix manual parser refresh reliability
All checks were successful
All checks were successful
This commit is contained in:
@@ -49,6 +49,35 @@ class GispProductsClientTest(SimpleTestCase):
|
||||
):
|
||||
GispProductsClient(max_records=10_001)
|
||||
|
||||
def test_retries_transient_server_error(self):
|
||||
calls = 0
|
||||
|
||||
def handle_page(_request, _body: bytes) -> Response:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
if calls == 1:
|
||||
return Response(status=504, body=b"gateway timeout")
|
||||
return Response(
|
||||
body=b'{"ok":true,"total_count":1,"items":[]}',
|
||||
headers={"Content-Type": "application/json"},
|
||||
)
|
||||
|
||||
with HTTPTestServer() as server:
|
||||
server.add_route("POST", "/pp719v2/pub/prod/b/", handle_page)
|
||||
client = GispProductsClient(
|
||||
base_url=server.base_url,
|
||||
max_pages=1,
|
||||
retry_backoff_seconds=0,
|
||||
http_adapter=server.adapter,
|
||||
)
|
||||
with self.assertRaisesMessage(
|
||||
GispProductsClientError,
|
||||
"empty page before total_count",
|
||||
):
|
||||
client.fetch_products()
|
||||
|
||||
self.assertEqual(calls, 2)
|
||||
|
||||
def test_fetch_products_follows_skip_pagination_and_maps_items(self):
|
||||
requests: list[dict] = []
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import tempfile
|
||||
import types
|
||||
from asyncio import events as asyncio_events
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from xml.etree import ElementPath as element_path
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
@@ -563,6 +564,18 @@ class ProverkiParseXMLTest(SimpleTestCase):
|
||||
inspections = client._parse_xml_content(xml)
|
||||
self.assertEqual(len(inspections), 1)
|
||||
|
||||
def test_large_xml_streaming_does_not_build_sanitized_string(self):
|
||||
xml = _xml_with_tag("INSPECTION", _inspection_attrs())
|
||||
client = ProverkiClient()
|
||||
client.STREAMING_THRESHOLD_BYTES = 1
|
||||
with patch.object(
|
||||
client,
|
||||
"_sanitize_xml",
|
||||
side_effect=AssertionError("large XML must stay as bytes"),
|
||||
):
|
||||
inspections = client._parse_xml_content(xml)
|
||||
self.assertEqual(len(inspections), 1)
|
||||
|
||||
def test_parse_xml_record_missing_fields_returns_none(self):
|
||||
element = ET.fromstring("<INSPECTION />") # noqa: S314
|
||||
client = ProverkiClient()
|
||||
|
||||
@@ -11,7 +11,7 @@ import zipfile
|
||||
from datetime import date
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from apps.core.models import BackgroundJob
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.parsers.models import (
|
||||
FinancialReport,
|
||||
FinancialReportLine,
|
||||
@@ -1428,3 +1428,22 @@ class ParsersViewSetTest(APITestCase):
|
||||
queued_task_id = apply_async_mock.call_args.kwargs["task_id"]
|
||||
job = BackgroundJob.objects.get(task_id=queued_task_id)
|
||||
self.assertEqual(job.meta["source_key"], "mpt_products")
|
||||
|
||||
def test_run_parser_reuses_fresh_active_job(self):
|
||||
self.client.force_authenticate(self.user)
|
||||
existing = BackgroundJob.objects.create(
|
||||
task_id="active-products",
|
||||
task_name="apps.parsers.tasks.parse_industrial_products",
|
||||
status=JobStatus.STARTED,
|
||||
)
|
||||
url = reverse("api_v1:parsers:run-parser", args=["industrial_products"])
|
||||
|
||||
with patch(
|
||||
"apps.parsers.views.tasks.parse_industrial_products.apply_async"
|
||||
) as apply_async_mock:
|
||||
response = self.client.post(url, {}, format="json")
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
|
||||
self.assertEqual(response.data["data"]["task_id"], existing.task_id)
|
||||
self.assertTrue(response.data["data"]["already_running"])
|
||||
apply_async_mock.assert_not_called()
|
||||
|
||||
Reference in New Issue
Block a user