"""Lease enrichment: real parsing/persistence, synthetic public-page transport.""" import importlib from contextlib import nullcontext from unittest.mock import Mock, patch import pytest from apps.parsers.fedresurs_company_profile import FedresursUpstreamError from django.core.cache import cache from django.utils import timezone from organizations.models import Organization, OrganizationSourceRecord from organizations.source_ingestion import ( OrganizationSourceIngestionService, SourceRecordInput, ) from rest_framework.test import APIClient from tests.apps.user.factories import UserFactory GUID = "12345678-1234-4234-8234-123456789abc" URL = f"https://fedresurs.ru/sfactmessages/{GUID}" NOT_FOUND_PAGE = ( "
404
Страница не найдена
" ) class RenderedMessageBrowser: """Replace browser transport; keep the actual wait, parsing and persistence.""" def __init__(self, pages): self.pages = pages self.page_source = "" self.current_url = "" def get(self, url): self.current_url = url self.page_source = self.pages[url] def set_page_load_timeout(self, timeout): pass def find_elements(self, by, value): return [object()] if "" in self.page_source else [] def page(kind="stop-financial-lease-contract-2", count=2): """Synthetic DOM based on the public message components inspected on 06.10.""" return f"""
12345678 от 17.09.2026 08:29:10 МСК
Договор
TEST-42 от 08.12.2025
Срок финансовой аренды
23.12.2025 - 15.09.2026
Дата прекращения
15.09.2026
Причина прекращения
Окончание договора
Лизингодатели
АО Тест 1
ИНН
7707083893
ОГРН
1027700132195
Лизингополучатели
АО Тест 2
ИНН
7702070139
ОГРН
1027700070518
Предметы финансовой аренды (лизинга) ({count})
Предмет финансовой аренды (лизинга)Описание
1
Идентификатор
нет данных
Классификатор
0104, Оборудование
Станок с ЧПУ
2
Идентификатор
OBJ-TEST
Классификатор
0104, Оборудование
Станок с ЧПУ
Заключение договора
""" def module(): try: return importlib.import_module("apps.parsers.leasing_details") except ImportError: pytest.fail("Lease detail enrichment is not implemented") @pytest.mark.parametrize( "kind", [ "financial-lease-contract-2", "change-financial-lease-contract-2", "stop-financial-lease-contract-2", "financial-lease-contract", ], ) def test_public_message_keeps_contract_parties_and_each_subject(kind): result = module().parse_leasing_html(page(kind), url=URL) assert result["contract_number"] == "TEST-42" assert result["contract_date"] == "2025-12-08" assert result["lease_start_date"] == "2025-12-23" assert result["lease_end_date"] == "2026-09-15" assert result["lessors"] == [ {"name": "АО Тест 1", "inn": "7707083893", "ogrn": "1027700132195"} ] assert result["lessees"][0]["inn"] == "7702070139" assert len(result["subjects"]) == 2 assert result["subjects"][0]["identifier"] is None assert result["subjects"][1]["identifier"] == "OBJ-TEST" assert result["subjects"][0]["description"] == "Станок с ЧПУ" assert result["termination_reason"] == "Окончание договора" assert ( result["related_messages"][0]["guid"] == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" ) @pytest.mark.parametrize( "url", [ "http://fedresurs.ru/sfactmessages/" + GUID, "https://fedresurs.ru.evil.test/sfactmessages/" + GUID, "https://user@fedresurs.ru/sfactmessages/" + GUID, "https://fedresurs.ru/companies/" + GUID, ], ) def test_only_exact_public_message_urls_are_accepted(url): with pytest.raises(FedresursUpstreamError, match="invalid_leasing_message_url"): module().parse_leasing_html(page(), url=url) def test_partial_subject_table_is_not_published_as_complete(): with pytest.raises(FedresursUpstreamError, match="incomplete_leasing_subjects"): module().parse_leasing_html(page(count=21), url=URL) def test_403_is_a_source_error_not_an_empty_contract(): with pytest.raises(FedresursUpstreamError, match="upstream_access_blocked"): module().parse_leasing_html("403 Forbidden", url=URL) def test_rendered_404_is_reported_without_waiting_for_a_card(settings): settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01 browser = RenderedMessageBrowser({URL: NOT_FOUND_PAGE}) with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"): module().fetch_leasing_html(browser, URL) def test_parser_reports_not_found_instead_of_incomplete_message(): with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"): module().parse_leasing_html(NOT_FOUND_PAGE, url=URL) def test_404_words_inside_a_contract_are_not_a_missing_page(): result = module().parse_leasing_html( page().replace("Станок с ЧПУ", "Модель 404. Страница не найдена в инструкции"), url=URL, ) assert len(result["subjects"]) == 2 def test_browser_receives_explicit_user_agent_and_closes(): from unittest.mock import Mock from apps.parsers.fedresurs_browser import public_browser driver = Mock() with patch( "apps.parsers.fedresurs_browser.webdriver.Chrome", return_value=driver ) as chrome, public_browser(user_agent="Mozilla/5.0 ECOS-LeasingCollector/1.0"): assert ( "--user-agent=Mozilla/5.0 ECOS-LeasingCollector/1.0" in chrome.call_args.kwargs["options"].arguments ) driver.quit.assert_called_once() def seed(org, guid=GUID): return OrganizationSourceIngestionService.save_records( source="checko_leasing", load_batch=1, records=[ SourceRecordInput( external_id=f"{org.inn}:{guid}", title="Заключение договора лизинга", organization_uid=org.uid, organization_name=org.name, inn=org.inn, ogrn=org.ogrn, record_type="leasing_contract", status="published", url=f"https://fedresurs.ru/sfactmessages/{guid}", payload={ "message_guid": guid, "message_type": "FinancialLeaseContract2", "lease_event_type": "FinancialLeaseContract2", "message_type_name": "Заключение договора лизинга", "publication_raw": "2026-09-17", "publication_date": "2026-09-17", "publisher_identifier": org.ogrn, "participant_identifiers": [org.ogrn], "source_url": f"https://fedresurs.ru/sfactmessages/{guid}", "fetched_at": timezone.now().isoformat(), "upstream": {"GUID": guid}, }, ) ], ) @pytest.fixture def organizations(db): cache.clear() return [ Organization.objects.create( name=f"АО Тест {n}", inn=inn, ogrn=ogrn, okpo=f"1234567{n}", directory_imported_at=timezone.now(), opk_registry_membership=True, ) for n, (inn, ogrn) in enumerate( [("7707083893", "1027700132195"), ("7702070139", "1027700070518")] ) ] def run_enrichment(fetch, **kwargs): m = module() with patch.object( m, "public_browser", return_value=nullcontext(Mock()) ), patch.object(m, "fetch_leasing_html", side_effect=fetch): return m.enrich_leasing_details(**kwargs) def test_one_page_per_guid_and_second_run_uses_persisted_details(organizations): for org in organizations: seed(org) before = list(OrganizationSourceRecord.objects.values_list("uid", "load_batch")) visits = [] def fetch(driver, url): visits.append(url) return page() first = run_enrichment(fetch, max_pages=1) second = run_enrichment(fetch, max_pages=1) assert first["pages_requested"] == 1 assert first["records_updated"] == 2 assert second["pages_requested"] == 0 assert len(visits) == 1 assert ( list(OrganizationSourceRecord.objects.values_list("uid", "load_batch")) == before ) for obj in OrganizationSourceRecord.objects.all(): assert obj.payload["lease_details"]["subjects"][1]["identifier"] == "OBJ-TEST" def test_page_budget_is_enforced_and_next_run_advances(organizations): seed(organizations[0]) seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb") first = run_enrichment(lambda driver, url: page(), max_pages=1) assert first["pages_requested"] == 1 assert ( OrganizationSourceRecord.objects.filter( payload__lease_details_status__status="loaded" ).count() == 1 ) second = run_enrichment(lambda driver, url: page(), max_pages=1) assert second["pages_requested"] == 1 assert ( OrganizationSourceRecord.objects.filter( payload__lease_details_status__status="loaded" ).count() == 2 ) def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations): seed(organizations[0]) seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb") def blocked(driver, url): raise FedresursUpstreamError("upstream_access_blocked") result = run_enrichment(blocked, max_pages=2) assert result["pages_requested"] == 1 assert result["error_code"] == "upstream_access_blocked" assert ( OrganizationSourceRecord.objects.filter( payload__lease_details_status__status="failed" ).count() == 1 ) again = run_enrichment( lambda driver, url: pytest.fail("Cooldown must prevent network access"), max_pages=2, ) assert again["pages_requested"] == 0 assert again["error_code"] == "upstream_cooldown" def test_missing_publication_does_not_stop_the_next_message(organizations, settings): settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01 seed(organizations[0]) next_guid = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb" seed(organizations[0], next_guid) browser = RenderedMessageBrowser( {URL: NOT_FOUND_PAGE, f"https://fedresurs.ru/sfactmessages/{next_guid}": page()} ) m = module() with patch.object(m, "public_browser", return_value=nullcontext(browser)): result = m.enrich_leasing_details(max_pages=2) assert result["pages_requested"] == 2 assert result["records_updated"] == 1 assert result["messages_failed"] == 1 assert result["error_code"] == "leasing_message_not_found" assert not cache.get(m.COOLDOWN_KEY) missing = OrganizationSourceRecord.objects.get(payload__message_guid=GUID) assert "lease_details" not in missing.payload assert ( missing.payload["lease_details_status"]["error_code"] == "leasing_message_not_found" ) loaded = OrganizationSourceRecord.objects.get(payload__message_guid=next_guid) assert len(loaded.payload["lease_details"]["subjects"]) == 2 def test_subject_identity_mismatch_never_enriches_another_organization(organizations): seed(organizations[0]) result = run_enrichment( lambda driver, url: page() .replace("7707083893", "9999999999") .replace("1027700132195", "1999999999999"), max_pages=1, ) obj = OrganizationSourceRecord.objects.get() assert "lease_details" not in obj.payload assert result["records_updated"] == 0 assert result["error_code"] == "leasing_party_mismatch" assert obj.payload["lease_details_status"]["error_code"] == "leasing_party_mismatch" def test_list_and_detail_expose_enriched_subjects_without_changing_old_payload( organizations, ): seed(organizations[0]) client = APIClient() client.force_authenticate(UserFactory.create_user()) obj = OrganizationSourceRecord.objects.get() url = f"/api/v2/organization-source-records/{obj.uid}/" old = client.get(url) assert old.status_code == 200, old.data assert "lease_details" not in old.data.get("data", old.data)["payload"] run_enrichment(lambda driver, url: page(), max_pages=1) detail = client.get(url) listing = client.get( "/api/v2/organization-source-records/", {"source": "checko_leasing"} ) assert detail.status_code == listing.status_code == 200 for body in [detail.data.get("data", detail.data), listing.data["data"][0]]: assert body["payload"]["lease_details"]["contract_number"] == "TEST-42" assert len(body["payload"]["lease_details"]["subjects"]) == 2 def test_refresh_api_details_mode_dispatches_without_loading_checko(organizations): from apps.parsers.tasks_registry_snapshots import parse_checko_leasing client = APIClient() client.force_authenticate(UserFactory.create_user(is_staff=True)) with patch.object(parse_checko_leasing, "apply_async") as dispatch: response = client.post( "/api/v1/sources/checko-leasing/refresh/", {"params": {"mode": "details"}}, format="json", ) assert response.status_code == 202, response.data assert dispatch.call_args.kwargs["kwargs"]["mode"] == "details" def test_refresh_rejects_invalid_mode_before_dispatch(organizations): client = APIClient() client.force_authenticate(UserFactory.create_user(is_staff=True)) response = client.post( "/api/v1/sources/checko-leasing/refresh/", {"params": {"mode": "unsupported"}}, format="json", ) assert response.status_code == 400 def test_detail_job_does_not_change_collection_success_or_use_checko(organizations): from apps.parsers.models import ParserLoadLog from apps.parsers.tasks_registry_snapshots import parse_checko_leasing seed(organizations[0]) m = module() with patch.object( m, "public_browser", return_value=nullcontext(Mock()) ), patch.object(m, "fetch_leasing_html", return_value=page()), patch( "apps.parsers.checko_section_snapshots._client", side_effect=AssertionError("No Checko calls in details mode"), ): result = parse_checko_leasing.apply(kwargs={"mode": "details"}).get() assert result["records_updated"] == 1 assert result["mode"] == "details" assert ParserLoadLog.objects.count() == 0 def test_next_collection_keeps_details_for_same_event(organizations): from tempfile import TemporaryFile from apps.parsers.checko_section_snapshots import _collect_organization from apps.parsers.models import ParserSourceArtifact seed(organizations[0]) run_enrichment(lambda driver, url: page(), max_pages=1) existing = OrganizationSourceRecord.objects.get() fresh = SourceRecordInput( external_id=existing.external_id, title="Договор", organization_name=organizations[0].name, record_type="leasing_contract", inn=organizations[0].inn, ogrn=organizations[0].ogrn, payload={"message_guid": GUID}, ) artifact = ParserSourceArtifact.objects.create(source="checko_leasing") with TemporaryFile() as raw, patch( "apps.parsers.checko_section_snapshots.collect_checko_section", return_value=iter([fresh]), ): _collect_organization( None, organizations[0], "checko_leasing", artifact, 0, raw ) staged = artifact.staged_records.get() assert ( staged.normalized_data["payload"]["lease_details"]["contract_number"] == "TEST-42" ) def test_parallel_enrichment_is_rejected_before_browser_start(organizations): seed(organizations[0]) cache.set(module().LOCK_KEY, "other-owner", timeout=3600) result = run_enrichment( lambda driver, url: pytest.fail("Second collector must not navigate"), max_pages=1, ) assert result["pages_requested"] == 0 assert result["error_code"] == "enrichment_already_running" def test_successful_empty_run_does_not_open_browser(organizations): m = module() with patch.object( m, "public_browser", side_effect=AssertionError("No browser for empty queue") ): assert m.enrich_leasing_details()["pages_requested"] == 0 def test_targeted_run_deduplicates_compact_and_dashed_guid(organizations): seed(organizations[0]) seed(organizations[1], GUID.replace("-", "").upper()) target = OrganizationSourceRecord.objects.get( extension__organization=organizations[0] ) result = run_enrichment( lambda driver, url: page(), max_pages=1, record_uid=str(target.uid) ) assert result["pages_requested"] == 1 assert result["records_updated"] == 2 def test_browser_startup_failure_has_explicit_result_and_cooldown(organizations): from selenium.common.exceptions import WebDriverException seed(organizations[0]) m = module() with patch.object( m, "public_browser", side_effect=WebDriverException("startup failed") ): result = m.enrich_leasing_details(max_pages=1) assert result["error_code"] == "browser_failure" assert result["pages_requested"] == 0 assert cache.get(m.COOLDOWN_KEY) assert not cache.get(m.LOCK_KEY) def test_non_admin_cannot_launch_details(organizations): client = APIClient() client.force_authenticate(UserFactory.create_user()) response = client.post( "/api/v1/sources/checko-leasing/refresh/", {"params": {"mode": "details"}}, format="json", ) assert response.status_code == 403