fix: continue leasing enrichment after missing Fedresurs messages
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m2s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m14s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m57s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m2s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m14s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m57s
This commit is contained in:
@@ -50,6 +50,9 @@ UID, load_batch, количество событий и данные Checko пр
|
|||||||
- Ошибка страницы откладывает её повтор на 24 часа. 403, challenge, timeout или
|
- Ошибка страницы откладывает её повтор на 24 часа. 403, challenge, timeout или
|
||||||
ошибка браузера останавливают порцию; отказ источника включает общий cooldown
|
ошибка браузера останавливают порцию; отказ источника включает общий cooldown
|
||||||
на 24 часа. Техническая ошибка не становится пустым договором.
|
на 24 часа. Техническая ошибка не становится пустым договором.
|
||||||
|
- Публичная страница «404 / Страница не найдена» (Федресурс возвращает HTTP 200)
|
||||||
|
даёт `leasing_message_not_found` для конкретной записи. Такая публикация
|
||||||
|
пропускается до следующей попытки, сбор продолжается без общего cooldown.
|
||||||
- Сбор ограничен 30 минутами и защищён общим cache-lock; одновременно запущенный
|
- Сбор ограничен 30 минутами и защищён общим cache-lock; одновременно запущенный
|
||||||
collector возвращает `enrichment_already_running` до загрузки страницы.
|
collector возвращает `enrichment_already_running` до загрузки страницы.
|
||||||
|
|
||||||
|
|||||||
@@ -129,10 +129,22 @@ def _subjects(content) -> list[dict]:
|
|||||||
return subjects
|
return subjects
|
||||||
|
|
||||||
|
|
||||||
def parse_leasing_html(html: str, *, url: str) -> dict:
|
def _checked_message_soup(html: str):
|
||||||
message_guid(url)
|
"""A missing Angular route renders HTTP 200 with a visible 404 error page."""
|
||||||
_challenge(html)
|
_challenge(html)
|
||||||
soup = BeautifulSoup(html, "html.parser")
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
if soup.find("sfact-message-card") is None:
|
||||||
|
for hidden in soup.find_all(("script", "style", "template", "noscript")):
|
||||||
|
hidden.decompose()
|
||||||
|
text = soup.get_text(" ", strip=True).casefold()
|
||||||
|
if re.search(r"\b404\s+страница\s+не\s+найдена\b", text):
|
||||||
|
raise FedresursUpstreamError("leasing_message_not_found")
|
||||||
|
return soup
|
||||||
|
|
||||||
|
|
||||||
|
def parse_leasing_html(html: str, *, url: str) -> dict:
|
||||||
|
message_guid(url)
|
||||||
|
soup = _checked_message_soup(html)
|
||||||
root = soup.find("sfact-message-card")
|
root = soup.find("sfact-message-card")
|
||||||
content = root.find(CONTENT_TAG) if root else None
|
content = root.find(CONTENT_TAG) if root else None
|
||||||
if content is None:
|
if content is None:
|
||||||
@@ -188,10 +200,10 @@ def fetch_leasing_html(driver, url: str) -> str:
|
|||||||
expected_guid = message_guid(url)
|
expected_guid = message_guid(url)
|
||||||
try:
|
try:
|
||||||
driver.get(url)
|
driver.get(url)
|
||||||
_challenge(driver.page_source)
|
_checked_message_soup(driver.page_source)
|
||||||
|
|
||||||
def ready(browser):
|
def ready(browser):
|
||||||
_challenge(browser.page_source)
|
_checked_message_soup(browser.page_source)
|
||||||
return browser.find_elements(By.TAG_NAME, "sfact-message-card")
|
return browser.find_elements(By.TAG_NAME, "sfact-message-card")
|
||||||
|
|
||||||
WebDriverWait(driver, min(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS, 45)).until(
|
WebDriverWait(driver, min(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS, 45)).until(
|
||||||
|
|||||||
@@ -19,6 +19,28 @@ from tests.apps.user.factories import UserFactory
|
|||||||
|
|
||||||
GUID = "12345678-1234-4234-8234-123456789abc"
|
GUID = "12345678-1234-4234-8234-123456789abc"
|
||||||
URL = f"https://fedresurs.ru/sfactmessages/{GUID}"
|
URL = f"https://fedresurs.ru/sfactmessages/{GUID}"
|
||||||
|
NOT_FOUND_PAGE = (
|
||||||
|
"<html><body><div>404</div><div>Страница не найдена</div></body></html>"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class RenderedMessageBrowser:
|
||||||
|
"""Replace browser transport; keep the actual wait, parsing and persistence."""
|
||||||
|
|
||||||
|
def __init__(self, pages):
|
||||||
|
self.pages = pages
|
||||||
|
self.page_source = ""
|
||||||
|
self.current_url = ""
|
||||||
|
|
||||||
|
def get(self, url):
|
||||||
|
self.current_url = url
|
||||||
|
self.page_source = self.pages[url]
|
||||||
|
|
||||||
|
def set_page_load_timeout(self, timeout):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def find_elements(self, by, value):
|
||||||
|
return [object()] if "<sfact-message-card>" in self.page_source else []
|
||||||
|
|
||||||
|
|
||||||
def page(kind="stop-financial-lease-contract-2", count=2):
|
def page(kind="stop-financial-lease-contract-2", count=2):
|
||||||
@@ -111,6 +133,26 @@ def test_403_is_a_source_error_not_an_empty_contract():
|
|||||||
module().parse_leasing_html("<html><body>403 Forbidden</body></html>", url=URL)
|
module().parse_leasing_html("<html><body>403 Forbidden</body></html>", url=URL)
|
||||||
|
|
||||||
|
|
||||||
|
def test_rendered_404_is_reported_without_waiting_for_a_card(settings):
|
||||||
|
settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01
|
||||||
|
browser = RenderedMessageBrowser({URL: NOT_FOUND_PAGE})
|
||||||
|
with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"):
|
||||||
|
module().fetch_leasing_html(browser, URL)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_reports_not_found_instead_of_incomplete_message():
|
||||||
|
with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"):
|
||||||
|
module().parse_leasing_html(NOT_FOUND_PAGE, url=URL)
|
||||||
|
|
||||||
|
|
||||||
|
def test_404_words_inside_a_contract_are_not_a_missing_page():
|
||||||
|
result = module().parse_leasing_html(
|
||||||
|
page().replace("Станок с ЧПУ", "Модель 404. Страница не найдена в инструкции"),
|
||||||
|
url=URL,
|
||||||
|
)
|
||||||
|
assert len(result["subjects"]) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_browser_receives_explicit_user_agent_and_closes():
|
def test_browser_receives_explicit_user_agent_and_closes():
|
||||||
from unittest.mock import Mock
|
from unittest.mock import Mock
|
||||||
|
|
||||||
@@ -255,6 +297,32 @@ def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations):
|
|||||||
assert again["error_code"] == "upstream_cooldown"
|
assert again["error_code"] == "upstream_cooldown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_publication_does_not_stop_the_next_message(organizations, settings):
|
||||||
|
settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01
|
||||||
|
seed(organizations[0])
|
||||||
|
next_guid = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
|
||||||
|
seed(organizations[0], next_guid)
|
||||||
|
browser = RenderedMessageBrowser(
|
||||||
|
{URL: NOT_FOUND_PAGE, f"https://fedresurs.ru/sfactmessages/{next_guid}": page()}
|
||||||
|
)
|
||||||
|
m = module()
|
||||||
|
with patch.object(m, "public_browser", return_value=nullcontext(browser)):
|
||||||
|
result = m.enrich_leasing_details(max_pages=2)
|
||||||
|
assert result["pages_requested"] == 2
|
||||||
|
assert result["records_updated"] == 1
|
||||||
|
assert result["messages_failed"] == 1
|
||||||
|
assert result["error_code"] == "leasing_message_not_found"
|
||||||
|
assert not cache.get(m.COOLDOWN_KEY)
|
||||||
|
missing = OrganizationSourceRecord.objects.get(payload__message_guid=GUID)
|
||||||
|
assert "lease_details" not in missing.payload
|
||||||
|
assert (
|
||||||
|
missing.payload["lease_details_status"]["error_code"]
|
||||||
|
== "leasing_message_not_found"
|
||||||
|
)
|
||||||
|
loaded = OrganizationSourceRecord.objects.get(payload__message_guid=next_guid)
|
||||||
|
assert len(loaded.payload["lease_details"]["subjects"]) == 2
|
||||||
|
|
||||||
|
|
||||||
def test_subject_identity_mismatch_never_enriches_another_organization(organizations):
|
def test_subject_identity_mismatch_never_enriches_another_organization(organizations):
|
||||||
seed(organizations[0])
|
seed(organizations[0])
|
||||||
result = run_enrichment(
|
result = run_enrichment(
|
||||||
|
|||||||
Reference in New Issue
Block a user