From 70dfc55f3d74c20e6f1779a60cb514515667c566 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Tue, 6 Oct 2026 15:48:34 +0200 Subject: [PATCH] fix: continue leasing enrichment after missing Fedresurs messages --- docs/sources/leasing-details.md | 3 + src/apps/parsers/leasing_details.py | 20 +++++-- tests/apps/parsers/test_leasing_details.py | 68 ++++++++++++++++++++++ 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/docs/sources/leasing-details.md b/docs/sources/leasing-details.md index ad0cb5c..10b9edd 100644 --- a/docs/sources/leasing-details.md +++ b/docs/sources/leasing-details.md @@ -50,6 +50,9 @@ UID, load_batch, количество событий и данные Checko пр - Ошибка страницы откладывает её повтор на 24 часа. 403, challenge, timeout или ошибка браузера останавливают порцию; отказ источника включает общий cooldown на 24 часа. Техническая ошибка не становится пустым договором. +- Публичная страница «404 / Страница не найдена» (Федресурс возвращает HTTP 200) + даёт `leasing_message_not_found` для конкретной записи. Такая публикация + пропускается до следующей попытки, сбор продолжается без общего cooldown. - Сбор ограничен 30 минутами и защищён общим cache-lock; одновременно запущенный collector возвращает `enrichment_already_running` до загрузки страницы. diff --git a/src/apps/parsers/leasing_details.py b/src/apps/parsers/leasing_details.py index e1edfac..d8237f7 100644 --- a/src/apps/parsers/leasing_details.py +++ b/src/apps/parsers/leasing_details.py @@ -129,10 +129,22 @@ def _subjects(content) -> list[dict]: return subjects -def parse_leasing_html(html: str, *, url: str) -> dict: - message_guid(url) +def _checked_message_soup(html: str): + """A missing Angular route renders HTTP 200 with a visible 404 error page.""" _challenge(html) soup = BeautifulSoup(html, "html.parser") + if soup.find("sfact-message-card") is None: + for hidden in soup.find_all(("script", "style", "template", "noscript")): + hidden.decompose() + text = soup.get_text(" ", strip=True).casefold() + if re.search(r"\b404\s+страница\s+не\s+найдена\b", text): + raise FedresursUpstreamError("leasing_message_not_found") + return soup + + +def parse_leasing_html(html: str, *, url: str) -> dict: + message_guid(url) + soup = _checked_message_soup(html) root = soup.find("sfact-message-card") content = root.find(CONTENT_TAG) if root else None if content is None: @@ -188,10 +200,10 @@ def fetch_leasing_html(driver, url: str) -> str: expected_guid = message_guid(url) try: driver.get(url) - _challenge(driver.page_source) + _checked_message_soup(driver.page_source) def ready(browser): - _challenge(browser.page_source) + _checked_message_soup(browser.page_source) return browser.find_elements(By.TAG_NAME, "sfact-message-card") WebDriverWait(driver, min(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS, 45)).until( diff --git a/tests/apps/parsers/test_leasing_details.py b/tests/apps/parsers/test_leasing_details.py index 88b207d..7c4ada1 100644 --- a/tests/apps/parsers/test_leasing_details.py +++ b/tests/apps/parsers/test_leasing_details.py @@ -19,6 +19,28 @@ from tests.apps.user.factories import UserFactory GUID = "12345678-1234-4234-8234-123456789abc" URL = f"https://fedresurs.ru/sfactmessages/{GUID}" +NOT_FOUND_PAGE = ( + "
404
Страница не найдена
" +) + + +class RenderedMessageBrowser: + """Replace browser transport; keep the actual wait, parsing and persistence.""" + + def __init__(self, pages): + self.pages = pages + self.page_source = "" + self.current_url = "" + + def get(self, url): + self.current_url = url + self.page_source = self.pages[url] + + def set_page_load_timeout(self, timeout): + pass + + def find_elements(self, by, value): + return [object()] if "" in self.page_source else [] def page(kind="stop-financial-lease-contract-2", count=2): @@ -111,6 +133,26 @@ def test_403_is_a_source_error_not_an_empty_contract(): module().parse_leasing_html("403 Forbidden", url=URL) +def test_rendered_404_is_reported_without_waiting_for_a_card(settings): + settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01 + browser = RenderedMessageBrowser({URL: NOT_FOUND_PAGE}) + with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"): + module().fetch_leasing_html(browser, URL) + + +def test_parser_reports_not_found_instead_of_incomplete_message(): + with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"): + module().parse_leasing_html(NOT_FOUND_PAGE, url=URL) + + +def test_404_words_inside_a_contract_are_not_a_missing_page(): + result = module().parse_leasing_html( + page().replace("Станок с ЧПУ", "Модель 404. Страница не найдена в инструкции"), + url=URL, + ) + assert len(result["subjects"]) == 2 + + def test_browser_receives_explicit_user_agent_and_closes(): from unittest.mock import Mock @@ -255,6 +297,32 @@ def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations): assert again["error_code"] == "upstream_cooldown" +def test_missing_publication_does_not_stop_the_next_message(organizations, settings): + settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01 + seed(organizations[0]) + next_guid = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" + seed(organizations[0], next_guid) + browser = RenderedMessageBrowser( + {URL: NOT_FOUND_PAGE, f"https://fedresurs.ru/sfactmessages/{next_guid}": page()} + ) + m = module() + with patch.object(m, "public_browser", return_value=nullcontext(browser)): + result = m.enrich_leasing_details(max_pages=2) + assert result["pages_requested"] == 2 + assert result["records_updated"] == 1 + assert result["messages_failed"] == 1 + assert result["error_code"] == "leasing_message_not_found" + assert not cache.get(m.COOLDOWN_KEY) + missing = OrganizationSourceRecord.objects.get(payload__message_guid=GUID) + assert "lease_details" not in missing.payload + assert ( + missing.payload["lease_details_status"]["error_code"] + == "leasing_message_not_found" + ) + loaded = OrganizationSourceRecord.objects.get(payload__message_guid=next_guid) + assert len(loaded.payload["lease_details"]["subjects"]) == 2 + + def test_subject_identity_mismatch_never_enriches_another_organization(organizations): seed(organizations[0]) result = run_enrichment(