Files
mostovik-backend/tests/apps/parsers/test_leasing_details.py
Aleksandr Meshchryakov 70dfc55f3d
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m2s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m14s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m57s
fix: continue leasing enrichment after missing Fedresurs messages
2026-10-06 15:48:34 +02:00

496 lines
19 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Lease enrichment: real parsing/persistence, synthetic public-page transport."""
import importlib
from contextlib import nullcontext
from unittest.mock import Mock, patch
import pytest
from apps.parsers.fedresurs_company_profile import FedresursUpstreamError
from django.core.cache import cache
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
GUID = "12345678-1234-4234-8234-123456789abc"
URL = f"https://fedresurs.ru/sfactmessages/{GUID}"
NOT_FOUND_PAGE = (
"<html><body><div>404</div><div>Страница не найдена</div></body></html>"
)
class RenderedMessageBrowser:
"""Replace browser transport; keep the actual wait, parsing and persistence."""
def __init__(self, pages):
self.pages = pages
self.page_source = ""
self.current_url = ""
def get(self, url):
self.current_url = url
self.page_source = self.pages[url]
def set_page_load_timeout(self, timeout):
pass
def find_elements(self, by, value):
return [object()] if "<sfact-message-card>" in self.page_source else []
def page(kind="stop-financial-lease-contract-2", count=2):
"""Synthetic DOM based on the public message components inspected on 06.10."""
return f"""<sfact-message-card>
<sfact-message-card-header><div class="header-item">12345678 от 17.09.2026 08:29:10 МСК</div></sfact-message-card-header>
<sfact-message-{kind}>
<div><div><span>Договор</span></div><div><span>TEST-42</span><span> от 08.12.2025</span></div></div>
<div><div>Срок финансовой аренды</div><div>23.12.2025 - 15.09.2026</div></div>
<div><div>Дата прекращения</div><div>15.09.2026</div></div>
<div><div>Причина прекращения</div><div>Окончание договора</div></div>
<div><div>Лизингодатели</div><div><message-participant><company-publisher>
<div class="name">АО Тест 1</div><div><div>ИНН</div><div>7707083893</div></div>
<div><div>ОГРН</div><div>1027700132195</div></div>
</company-publisher></message-participant></div></div>
<div><div>Лизингополучатели</div><div><message-participant><company-publisher>
<div class="name">АО Тест 2</div><div><div>ИНН</div><div>7702070139</div></div>
<div><div>ОГРН</div><div>1027700070518</div></div>
</company-publisher></message-participant></div></div>
<div>Предметы финансовой аренды (лизинга) ({count})</div>
<sfact-message-subject-contract-table-2><table><thead><tr><th></th>
<th>Предмет финансовой аренды (лизинга)</th><th>Описание</th></tr></thead><tbody>
<tr><td>1</td><td><div><div>Идентификатор</div><div>нет данных</div></div>
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
<tr><td>2</td><td><div><div>Идентификатор</div><div>OBJ-TEST</div></div>
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
</tbody></table></sfact-message-subject-contract-table-2>
</sfact-message-{kind}>
<sfact-message-card-msgs><a href="/sfactmessage/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa">Заключение договора</a></sfact-message-card-msgs>
</sfact-message-card>"""
def module():
try:
return importlib.import_module("apps.parsers.leasing_details")
except ImportError:
pytest.fail("Lease detail enrichment is not implemented")
@pytest.mark.parametrize(
"kind",
[
"financial-lease-contract-2",
"change-financial-lease-contract-2",
"stop-financial-lease-contract-2",
"financial-lease-contract",
],
)
def test_public_message_keeps_contract_parties_and_each_subject(kind):
result = module().parse_leasing_html(page(kind), url=URL)
assert result["contract_number"] == "TEST-42"
assert result["contract_date"] == "2025-12-08"
assert result["lease_start_date"] == "2025-12-23"
assert result["lease_end_date"] == "2026-09-15"
assert result["lessors"] == [
{"name": "АО Тест 1", "inn": "7707083893", "ogrn": "1027700132195"}
]
assert result["lessees"][0]["inn"] == "7702070139"
assert len(result["subjects"]) == 2
assert result["subjects"][0]["identifier"] is None
assert result["subjects"][1]["identifier"] == "OBJ-TEST"
assert result["subjects"][0]["description"] == "Станок с ЧПУ"
assert result["termination_reason"] == "Окончание договора"
assert (
result["related_messages"][0]["guid"] == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
)
@pytest.mark.parametrize(
"url",
[
"http://fedresurs.ru/sfactmessages/" + GUID,
"https://fedresurs.ru.evil.test/sfactmessages/" + GUID,
"https://user@fedresurs.ru/sfactmessages/" + GUID,
"https://fedresurs.ru/companies/" + GUID,
],
)
def test_only_exact_public_message_urls_are_accepted(url):
with pytest.raises(FedresursUpstreamError, match="invalid_leasing_message_url"):
module().parse_leasing_html(page(), url=url)
def test_partial_subject_table_is_not_published_as_complete():
with pytest.raises(FedresursUpstreamError, match="incomplete_leasing_subjects"):
module().parse_leasing_html(page(count=21), url=URL)
def test_403_is_a_source_error_not_an_empty_contract():
with pytest.raises(FedresursUpstreamError, match="upstream_access_blocked"):
module().parse_leasing_html("<html><body>403 Forbidden</body></html>", url=URL)
def test_rendered_404_is_reported_without_waiting_for_a_card(settings):
settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01
browser = RenderedMessageBrowser({URL: NOT_FOUND_PAGE})
with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"):
module().fetch_leasing_html(browser, URL)
def test_parser_reports_not_found_instead_of_incomplete_message():
with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"):
module().parse_leasing_html(NOT_FOUND_PAGE, url=URL)
def test_404_words_inside_a_contract_are_not_a_missing_page():
result = module().parse_leasing_html(
page().replace("Станок с ЧПУ", "Модель 404. Страница не найдена в инструкции"),
url=URL,
)
assert len(result["subjects"]) == 2
def test_browser_receives_explicit_user_agent_and_closes():
from unittest.mock import Mock
from apps.parsers.fedresurs_browser import public_browser
driver = Mock()
with patch(
"apps.parsers.fedresurs_browser.webdriver.Chrome", return_value=driver
) as chrome, public_browser(user_agent="Mozilla/5.0 ECOS-LeasingCollector/1.0"):
assert (
"--user-agent=Mozilla/5.0 ECOS-LeasingCollector/1.0"
in chrome.call_args.kwargs["options"].arguments
)
driver.quit.assert_called_once()
def seed(org, guid=GUID):
return OrganizationSourceIngestionService.save_records(
source="checko_leasing",
load_batch=1,
records=[
SourceRecordInput(
external_id=f"{org.inn}:{guid}",
title="Заключение договора лизинга",
organization_uid=org.uid,
organization_name=org.name,
inn=org.inn,
ogrn=org.ogrn,
record_type="leasing_contract",
status="published",
url=f"https://fedresurs.ru/sfactmessages/{guid}",
payload={
"message_guid": guid,
"message_type": "FinancialLeaseContract2",
"lease_event_type": "FinancialLeaseContract2",
"message_type_name": "Заключение договора лизинга",
"publication_raw": "2026-09-17",
"publication_date": "2026-09-17",
"publisher_identifier": org.ogrn,
"participant_identifiers": [org.ogrn],
"source_url": f"https://fedresurs.ru/sfactmessages/{guid}",
"fetched_at": timezone.now().isoformat(),
"upstream": {"GUID": guid},
},
)
],
)
@pytest.fixture
def organizations(db):
cache.clear()
return [
Organization.objects.create(
name=f"АО Тест {n}",
inn=inn,
ogrn=ogrn,
okpo=f"1234567{n}",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
for n, (inn, ogrn) in enumerate(
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
)
]
def run_enrichment(fetch, **kwargs):
m = module()
with patch.object(
m, "public_browser", return_value=nullcontext(Mock())
), patch.object(m, "fetch_leasing_html", side_effect=fetch):
return m.enrich_leasing_details(**kwargs)
def test_one_page_per_guid_and_second_run_uses_persisted_details(organizations):
for org in organizations:
seed(org)
before = list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
visits = []
def fetch(driver, url):
visits.append(url)
return page()
first = run_enrichment(fetch, max_pages=1)
second = run_enrichment(fetch, max_pages=1)
assert first["pages_requested"] == 1
assert first["records_updated"] == 2
assert second["pages_requested"] == 0
assert len(visits) == 1
assert (
list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
== before
)
for obj in OrganizationSourceRecord.objects.all():
assert obj.payload["lease_details"]["subjects"][1]["identifier"] == "OBJ-TEST"
def test_page_budget_is_enforced_and_next_run_advances(organizations):
seed(organizations[0])
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
first = run_enrichment(lambda driver, url: page(), max_pages=1)
assert first["pages_requested"] == 1
assert (
OrganizationSourceRecord.objects.filter(
payload__lease_details_status__status="loaded"
).count()
== 1
)
second = run_enrichment(lambda driver, url: page(), max_pages=1)
assert second["pages_requested"] == 1
assert (
OrganizationSourceRecord.objects.filter(
payload__lease_details_status__status="loaded"
).count()
== 2
)
def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations):
seed(organizations[0])
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
def blocked(driver, url):
raise FedresursUpstreamError("upstream_access_blocked")
result = run_enrichment(blocked, max_pages=2)
assert result["pages_requested"] == 1
assert result["error_code"] == "upstream_access_blocked"
assert (
OrganizationSourceRecord.objects.filter(
payload__lease_details_status__status="failed"
).count()
== 1
)
again = run_enrichment(
lambda driver, url: pytest.fail("Cooldown must prevent network access"),
max_pages=2,
)
assert again["pages_requested"] == 0
assert again["error_code"] == "upstream_cooldown"
def test_missing_publication_does_not_stop_the_next_message(organizations, settings):
settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01
seed(organizations[0])
next_guid = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
seed(organizations[0], next_guid)
browser = RenderedMessageBrowser(
{URL: NOT_FOUND_PAGE, f"https://fedresurs.ru/sfactmessages/{next_guid}": page()}
)
m = module()
with patch.object(m, "public_browser", return_value=nullcontext(browser)):
result = m.enrich_leasing_details(max_pages=2)
assert result["pages_requested"] == 2
assert result["records_updated"] == 1
assert result["messages_failed"] == 1
assert result["error_code"] == "leasing_message_not_found"
assert not cache.get(m.COOLDOWN_KEY)
missing = OrganizationSourceRecord.objects.get(payload__message_guid=GUID)
assert "lease_details" not in missing.payload
assert (
missing.payload["lease_details_status"]["error_code"]
== "leasing_message_not_found"
)
loaded = OrganizationSourceRecord.objects.get(payload__message_guid=next_guid)
assert len(loaded.payload["lease_details"]["subjects"]) == 2
def test_subject_identity_mismatch_never_enriches_another_organization(organizations):
seed(organizations[0])
result = run_enrichment(
lambda driver, url: page()
.replace("7707083893", "9999999999")
.replace("1027700132195", "1999999999999"),
max_pages=1,
)
obj = OrganizationSourceRecord.objects.get()
assert "lease_details" not in obj.payload
assert result["records_updated"] == 0
assert result["error_code"] == "leasing_party_mismatch"
assert obj.payload["lease_details_status"]["error_code"] == "leasing_party_mismatch"
def test_list_and_detail_expose_enriched_subjects_without_changing_old_payload(
organizations,
):
seed(organizations[0])
client = APIClient()
client.force_authenticate(UserFactory.create_user())
obj = OrganizationSourceRecord.objects.get()
url = f"/api/v2/organization-source-records/{obj.uid}/"
old = client.get(url)
assert old.status_code == 200, old.data
assert "lease_details" not in old.data.get("data", old.data)["payload"]
run_enrichment(lambda driver, url: page(), max_pages=1)
detail = client.get(url)
listing = client.get(
"/api/v2/organization-source-records/", {"source": "checko_leasing"}
)
assert detail.status_code == listing.status_code == 200
for body in [detail.data.get("data", detail.data), listing.data["data"][0]]:
assert body["payload"]["lease_details"]["contract_number"] == "TEST-42"
assert len(body["payload"]["lease_details"]["subjects"]) == 2
def test_refresh_api_details_mode_dispatches_without_loading_checko(organizations):
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
client = APIClient()
client.force_authenticate(UserFactory.create_user(is_staff=True))
with patch.object(parse_checko_leasing, "apply_async") as dispatch:
response = client.post(
"/api/v1/sources/checko-leasing/refresh/",
{"params": {"mode": "details"}},
format="json",
)
assert response.status_code == 202, response.data
assert dispatch.call_args.kwargs["kwargs"]["mode"] == "details"
def test_refresh_rejects_invalid_mode_before_dispatch(organizations):
client = APIClient()
client.force_authenticate(UserFactory.create_user(is_staff=True))
response = client.post(
"/api/v1/sources/checko-leasing/refresh/",
{"params": {"mode": "unsupported"}},
format="json",
)
assert response.status_code == 400
def test_detail_job_does_not_change_collection_success_or_use_checko(organizations):
from apps.parsers.models import ParserLoadLog
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
seed(organizations[0])
m = module()
with patch.object(
m, "public_browser", return_value=nullcontext(Mock())
), patch.object(m, "fetch_leasing_html", return_value=page()), patch(
"apps.parsers.checko_section_snapshots._client",
side_effect=AssertionError("No Checko calls in details mode"),
):
result = parse_checko_leasing.apply(kwargs={"mode": "details"}).get()
assert result["records_updated"] == 1
assert result["mode"] == "details"
assert ParserLoadLog.objects.count() == 0
def test_next_collection_keeps_details_for_same_event(organizations):
from tempfile import TemporaryFile
from apps.parsers.checko_section_snapshots import _collect_organization
from apps.parsers.models import ParserSourceArtifact
seed(organizations[0])
run_enrichment(lambda driver, url: page(), max_pages=1)
existing = OrganizationSourceRecord.objects.get()
fresh = SourceRecordInput(
external_id=existing.external_id,
title="Договор",
organization_name=organizations[0].name,
record_type="leasing_contract",
inn=organizations[0].inn,
ogrn=organizations[0].ogrn,
payload={"message_guid": GUID},
)
artifact = ParserSourceArtifact.objects.create(source="checko_leasing")
with TemporaryFile() as raw, patch(
"apps.parsers.checko_section_snapshots.collect_checko_section",
return_value=iter([fresh]),
):
_collect_organization(
None, organizations[0], "checko_leasing", artifact, 0, raw
)
staged = artifact.staged_records.get()
assert (
staged.normalized_data["payload"]["lease_details"]["contract_number"]
== "TEST-42"
)
def test_parallel_enrichment_is_rejected_before_browser_start(organizations):
seed(organizations[0])
cache.set(module().LOCK_KEY, "other-owner", timeout=3600)
result = run_enrichment(
lambda driver, url: pytest.fail("Second collector must not navigate"),
max_pages=1,
)
assert result["pages_requested"] == 0
assert result["error_code"] == "enrichment_already_running"
def test_successful_empty_run_does_not_open_browser(organizations):
m = module()
with patch.object(
m, "public_browser", side_effect=AssertionError("No browser for empty queue")
):
assert m.enrich_leasing_details()["pages_requested"] == 0
def test_targeted_run_deduplicates_compact_and_dashed_guid(organizations):
seed(organizations[0])
seed(organizations[1], GUID.replace("-", "").upper())
target = OrganizationSourceRecord.objects.get(
extension__organization=organizations[0]
)
result = run_enrichment(
lambda driver, url: page(), max_pages=1, record_uid=str(target.uid)
)
assert result["pages_requested"] == 1
assert result["records_updated"] == 2
def test_browser_startup_failure_has_explicit_result_and_cooldown(organizations):
from selenium.common.exceptions import WebDriverException
seed(organizations[0])
m = module()
with patch.object(
m, "public_browser", side_effect=WebDriverException("startup failed")
):
result = m.enrich_leasing_details(max_pages=1)
assert result["error_code"] == "browser_failure"
assert result["pages_requested"] == 0
assert cache.get(m.COOLDOWN_KEY)
assert not cache.get(m.LOCK_KEY)
def test_non_admin_cannot_launch_details(organizations):
client = APIClient()
client.force_authenticate(UserFactory.create_user())
response = client.post(
"/api/v1/sources/checko-leasing/refresh/",
{"params": {"mode": "details"}},
format="json",
)
assert response.status_code == 403