fix: continue leasing enrichment after missing Fedresurs messages
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m2s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m14s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m57s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m2s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m14s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m57s
This commit is contained in:
@@ -129,10 +129,22 @@ def _subjects(content) -> list[dict]:
|
||||
return subjects
|
||||
|
||||
|
||||
def parse_leasing_html(html: str, *, url: str) -> dict:
|
||||
message_guid(url)
|
||||
def _checked_message_soup(html: str):
|
||||
"""A missing Angular route renders HTTP 200 with a visible 404 error page."""
|
||||
_challenge(html)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
if soup.find("sfact-message-card") is None:
|
||||
for hidden in soup.find_all(("script", "style", "template", "noscript")):
|
||||
hidden.decompose()
|
||||
text = soup.get_text(" ", strip=True).casefold()
|
||||
if re.search(r"\b404\s+страница\s+не\s+найдена\b", text):
|
||||
raise FedresursUpstreamError("leasing_message_not_found")
|
||||
return soup
|
||||
|
||||
|
||||
def parse_leasing_html(html: str, *, url: str) -> dict:
|
||||
message_guid(url)
|
||||
soup = _checked_message_soup(html)
|
||||
root = soup.find("sfact-message-card")
|
||||
content = root.find(CONTENT_TAG) if root else None
|
||||
if content is None:
|
||||
@@ -188,10 +200,10 @@ def fetch_leasing_html(driver, url: str) -> str:
|
||||
expected_guid = message_guid(url)
|
||||
try:
|
||||
driver.get(url)
|
||||
_challenge(driver.page_source)
|
||||
_checked_message_soup(driver.page_source)
|
||||
|
||||
def ready(browser):
|
||||
_challenge(browser.page_source)
|
||||
_checked_message_soup(browser.page_source)
|
||||
return browser.find_elements(By.TAG_NAME, "sfact-message-card")
|
||||
|
||||
WebDriverWait(driver, min(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS, 45)).until(
|
||||
|
||||
Reference in New Issue
Block a user