All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m2s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m14s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m57s
496 lines
19 KiB
Python
496 lines
19 KiB
Python
"""Lease enrichment: real parsing/persistence, synthetic public-page transport."""
|
||
|
||
import importlib
|
||
from contextlib import nullcontext
|
||
from unittest.mock import Mock, patch
|
||
|
||
import pytest
|
||
from apps.parsers.fedresurs_company_profile import FedresursUpstreamError
|
||
from django.core.cache import cache
|
||
from django.utils import timezone
|
||
from organizations.models import Organization, OrganizationSourceRecord
|
||
from organizations.source_ingestion import (
|
||
OrganizationSourceIngestionService,
|
||
SourceRecordInput,
|
||
)
|
||
from rest_framework.test import APIClient
|
||
|
||
from tests.apps.user.factories import UserFactory
|
||
|
||
GUID = "12345678-1234-4234-8234-123456789abc"
|
||
URL = f"https://fedresurs.ru/sfactmessages/{GUID}"
|
||
NOT_FOUND_PAGE = (
|
||
"<html><body><div>404</div><div>Страница не найдена</div></body></html>"
|
||
)
|
||
|
||
|
||
class RenderedMessageBrowser:
|
||
"""Replace browser transport; keep the actual wait, parsing and persistence."""
|
||
|
||
def __init__(self, pages):
|
||
self.pages = pages
|
||
self.page_source = ""
|
||
self.current_url = ""
|
||
|
||
def get(self, url):
|
||
self.current_url = url
|
||
self.page_source = self.pages[url]
|
||
|
||
def set_page_load_timeout(self, timeout):
|
||
pass
|
||
|
||
def find_elements(self, by, value):
|
||
return [object()] if "<sfact-message-card>" in self.page_source else []
|
||
|
||
|
||
def page(kind="stop-financial-lease-contract-2", count=2):
|
||
"""Synthetic DOM based on the public message components inspected on 06.10."""
|
||
return f"""<sfact-message-card>
|
||
<sfact-message-card-header><div class="header-item">12345678 от 17.09.2026 08:29:10 МСК</div></sfact-message-card-header>
|
||
<sfact-message-{kind}>
|
||
<div><div><span>Договор</span></div><div><span>TEST-42</span><span> от 08.12.2025</span></div></div>
|
||
<div><div>Срок финансовой аренды</div><div>23.12.2025 - 15.09.2026</div></div>
|
||
<div><div>Дата прекращения</div><div>15.09.2026</div></div>
|
||
<div><div>Причина прекращения</div><div>Окончание договора</div></div>
|
||
<div><div>Лизингодатели</div><div><message-participant><company-publisher>
|
||
<div class="name">АО Тест 1</div><div><div>ИНН</div><div>7707083893</div></div>
|
||
<div><div>ОГРН</div><div>1027700132195</div></div>
|
||
</company-publisher></message-participant></div></div>
|
||
<div><div>Лизингополучатели</div><div><message-participant><company-publisher>
|
||
<div class="name">АО Тест 2</div><div><div>ИНН</div><div>7702070139</div></div>
|
||
<div><div>ОГРН</div><div>1027700070518</div></div>
|
||
</company-publisher></message-participant></div></div>
|
||
<div>Предметы финансовой аренды (лизинга) ({count})</div>
|
||
<sfact-message-subject-contract-table-2><table><thead><tr><th></th>
|
||
<th>Предмет финансовой аренды (лизинга)</th><th>Описание</th></tr></thead><tbody>
|
||
<tr><td>1</td><td><div><div>Идентификатор</div><div>нет данных</div></div>
|
||
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
|
||
<tr><td>2</td><td><div><div>Идентификатор</div><div>OBJ-TEST</div></div>
|
||
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
|
||
</tbody></table></sfact-message-subject-contract-table-2>
|
||
</sfact-message-{kind}>
|
||
<sfact-message-card-msgs><a href="/sfactmessage/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa">Заключение договора</a></sfact-message-card-msgs>
|
||
</sfact-message-card>"""
|
||
|
||
|
||
def module():
|
||
try:
|
||
return importlib.import_module("apps.parsers.leasing_details")
|
||
except ImportError:
|
||
pytest.fail("Lease detail enrichment is not implemented")
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"kind",
|
||
[
|
||
"financial-lease-contract-2",
|
||
"change-financial-lease-contract-2",
|
||
"stop-financial-lease-contract-2",
|
||
"financial-lease-contract",
|
||
],
|
||
)
|
||
def test_public_message_keeps_contract_parties_and_each_subject(kind):
|
||
result = module().parse_leasing_html(page(kind), url=URL)
|
||
assert result["contract_number"] == "TEST-42"
|
||
assert result["contract_date"] == "2025-12-08"
|
||
assert result["lease_start_date"] == "2025-12-23"
|
||
assert result["lease_end_date"] == "2026-09-15"
|
||
assert result["lessors"] == [
|
||
{"name": "АО Тест 1", "inn": "7707083893", "ogrn": "1027700132195"}
|
||
]
|
||
assert result["lessees"][0]["inn"] == "7702070139"
|
||
assert len(result["subjects"]) == 2
|
||
assert result["subjects"][0]["identifier"] is None
|
||
assert result["subjects"][1]["identifier"] == "OBJ-TEST"
|
||
assert result["subjects"][0]["description"] == "Станок с ЧПУ"
|
||
assert result["termination_reason"] == "Окончание договора"
|
||
assert (
|
||
result["related_messages"][0]["guid"] == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
||
)
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
"url",
|
||
[
|
||
"http://fedresurs.ru/sfactmessages/" + GUID,
|
||
"https://fedresurs.ru.evil.test/sfactmessages/" + GUID,
|
||
"https://user@fedresurs.ru/sfactmessages/" + GUID,
|
||
"https://fedresurs.ru/companies/" + GUID,
|
||
],
|
||
)
|
||
def test_only_exact_public_message_urls_are_accepted(url):
|
||
with pytest.raises(FedresursUpstreamError, match="invalid_leasing_message_url"):
|
||
module().parse_leasing_html(page(), url=url)
|
||
|
||
|
||
def test_partial_subject_table_is_not_published_as_complete():
|
||
with pytest.raises(FedresursUpstreamError, match="incomplete_leasing_subjects"):
|
||
module().parse_leasing_html(page(count=21), url=URL)
|
||
|
||
|
||
def test_403_is_a_source_error_not_an_empty_contract():
|
||
with pytest.raises(FedresursUpstreamError, match="upstream_access_blocked"):
|
||
module().parse_leasing_html("<html><body>403 Forbidden</body></html>", url=URL)
|
||
|
||
|
||
def test_rendered_404_is_reported_without_waiting_for_a_card(settings):
|
||
settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01
|
||
browser = RenderedMessageBrowser({URL: NOT_FOUND_PAGE})
|
||
with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"):
|
||
module().fetch_leasing_html(browser, URL)
|
||
|
||
|
||
def test_parser_reports_not_found_instead_of_incomplete_message():
|
||
with pytest.raises(FedresursUpstreamError, match="^leasing_message_not_found$"):
|
||
module().parse_leasing_html(NOT_FOUND_PAGE, url=URL)
|
||
|
||
|
||
def test_404_words_inside_a_contract_are_not_a_missing_page():
|
||
result = module().parse_leasing_html(
|
||
page().replace("Станок с ЧПУ", "Модель 404. Страница не найдена в инструкции"),
|
||
url=URL,
|
||
)
|
||
assert len(result["subjects"]) == 2
|
||
|
||
|
||
def test_browser_receives_explicit_user_agent_and_closes():
|
||
from unittest.mock import Mock
|
||
|
||
from apps.parsers.fedresurs_browser import public_browser
|
||
|
||
driver = Mock()
|
||
with patch(
|
||
"apps.parsers.fedresurs_browser.webdriver.Chrome", return_value=driver
|
||
) as chrome, public_browser(user_agent="Mozilla/5.0 ECOS-LeasingCollector/1.0"):
|
||
assert (
|
||
"--user-agent=Mozilla/5.0 ECOS-LeasingCollector/1.0"
|
||
in chrome.call_args.kwargs["options"].arguments
|
||
)
|
||
driver.quit.assert_called_once()
|
||
|
||
|
||
def seed(org, guid=GUID):
|
||
return OrganizationSourceIngestionService.save_records(
|
||
source="checko_leasing",
|
||
load_batch=1,
|
||
records=[
|
||
SourceRecordInput(
|
||
external_id=f"{org.inn}:{guid}",
|
||
title="Заключение договора лизинга",
|
||
organization_uid=org.uid,
|
||
organization_name=org.name,
|
||
inn=org.inn,
|
||
ogrn=org.ogrn,
|
||
record_type="leasing_contract",
|
||
status="published",
|
||
url=f"https://fedresurs.ru/sfactmessages/{guid}",
|
||
payload={
|
||
"message_guid": guid,
|
||
"message_type": "FinancialLeaseContract2",
|
||
"lease_event_type": "FinancialLeaseContract2",
|
||
"message_type_name": "Заключение договора лизинга",
|
||
"publication_raw": "2026-09-17",
|
||
"publication_date": "2026-09-17",
|
||
"publisher_identifier": org.ogrn,
|
||
"participant_identifiers": [org.ogrn],
|
||
"source_url": f"https://fedresurs.ru/sfactmessages/{guid}",
|
||
"fetched_at": timezone.now().isoformat(),
|
||
"upstream": {"GUID": guid},
|
||
},
|
||
)
|
||
],
|
||
)
|
||
|
||
|
||
@pytest.fixture
|
||
def organizations(db):
|
||
cache.clear()
|
||
return [
|
||
Organization.objects.create(
|
||
name=f"АО Тест {n}",
|
||
inn=inn,
|
||
ogrn=ogrn,
|
||
okpo=f"1234567{n}",
|
||
directory_imported_at=timezone.now(),
|
||
opk_registry_membership=True,
|
||
)
|
||
for n, (inn, ogrn) in enumerate(
|
||
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
|
||
)
|
||
]
|
||
|
||
|
||
def run_enrichment(fetch, **kwargs):
|
||
m = module()
|
||
with patch.object(
|
||
m, "public_browser", return_value=nullcontext(Mock())
|
||
), patch.object(m, "fetch_leasing_html", side_effect=fetch):
|
||
return m.enrich_leasing_details(**kwargs)
|
||
|
||
|
||
def test_one_page_per_guid_and_second_run_uses_persisted_details(organizations):
|
||
for org in organizations:
|
||
seed(org)
|
||
before = list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
|
||
visits = []
|
||
|
||
def fetch(driver, url):
|
||
visits.append(url)
|
||
return page()
|
||
|
||
first = run_enrichment(fetch, max_pages=1)
|
||
second = run_enrichment(fetch, max_pages=1)
|
||
assert first["pages_requested"] == 1
|
||
assert first["records_updated"] == 2
|
||
assert second["pages_requested"] == 0
|
||
assert len(visits) == 1
|
||
assert (
|
||
list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
|
||
== before
|
||
)
|
||
for obj in OrganizationSourceRecord.objects.all():
|
||
assert obj.payload["lease_details"]["subjects"][1]["identifier"] == "OBJ-TEST"
|
||
|
||
|
||
def test_page_budget_is_enforced_and_next_run_advances(organizations):
|
||
seed(organizations[0])
|
||
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||
first = run_enrichment(lambda driver, url: page(), max_pages=1)
|
||
assert first["pages_requested"] == 1
|
||
assert (
|
||
OrganizationSourceRecord.objects.filter(
|
||
payload__lease_details_status__status="loaded"
|
||
).count()
|
||
== 1
|
||
)
|
||
second = run_enrichment(lambda driver, url: page(), max_pages=1)
|
||
assert second["pages_requested"] == 1
|
||
assert (
|
||
OrganizationSourceRecord.objects.filter(
|
||
payload__lease_details_status__status="loaded"
|
||
).count()
|
||
== 2
|
||
)
|
||
|
||
|
||
def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations):
|
||
seed(organizations[0])
|
||
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||
|
||
def blocked(driver, url):
|
||
raise FedresursUpstreamError("upstream_access_blocked")
|
||
|
||
result = run_enrichment(blocked, max_pages=2)
|
||
assert result["pages_requested"] == 1
|
||
assert result["error_code"] == "upstream_access_blocked"
|
||
assert (
|
||
OrganizationSourceRecord.objects.filter(
|
||
payload__lease_details_status__status="failed"
|
||
).count()
|
||
== 1
|
||
)
|
||
again = run_enrichment(
|
||
lambda driver, url: pytest.fail("Cooldown must prevent network access"),
|
||
max_pages=2,
|
||
)
|
||
assert again["pages_requested"] == 0
|
||
assert again["error_code"] == "upstream_cooldown"
|
||
|
||
|
||
def test_missing_publication_does_not_stop_the_next_message(organizations, settings):
|
||
settings.FEDRESURS_PAGE_TIMEOUT_SECONDS = 0.01
|
||
seed(organizations[0])
|
||
next_guid = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"
|
||
seed(organizations[0], next_guid)
|
||
browser = RenderedMessageBrowser(
|
||
{URL: NOT_FOUND_PAGE, f"https://fedresurs.ru/sfactmessages/{next_guid}": page()}
|
||
)
|
||
m = module()
|
||
with patch.object(m, "public_browser", return_value=nullcontext(browser)):
|
||
result = m.enrich_leasing_details(max_pages=2)
|
||
assert result["pages_requested"] == 2
|
||
assert result["records_updated"] == 1
|
||
assert result["messages_failed"] == 1
|
||
assert result["error_code"] == "leasing_message_not_found"
|
||
assert not cache.get(m.COOLDOWN_KEY)
|
||
missing = OrganizationSourceRecord.objects.get(payload__message_guid=GUID)
|
||
assert "lease_details" not in missing.payload
|
||
assert (
|
||
missing.payload["lease_details_status"]["error_code"]
|
||
== "leasing_message_not_found"
|
||
)
|
||
loaded = OrganizationSourceRecord.objects.get(payload__message_guid=next_guid)
|
||
assert len(loaded.payload["lease_details"]["subjects"]) == 2
|
||
|
||
|
||
def test_subject_identity_mismatch_never_enriches_another_organization(organizations):
|
||
seed(organizations[0])
|
||
result = run_enrichment(
|
||
lambda driver, url: page()
|
||
.replace("7707083893", "9999999999")
|
||
.replace("1027700132195", "1999999999999"),
|
||
max_pages=1,
|
||
)
|
||
obj = OrganizationSourceRecord.objects.get()
|
||
assert "lease_details" not in obj.payload
|
||
assert result["records_updated"] == 0
|
||
assert result["error_code"] == "leasing_party_mismatch"
|
||
assert obj.payload["lease_details_status"]["error_code"] == "leasing_party_mismatch"
|
||
|
||
|
||
def test_list_and_detail_expose_enriched_subjects_without_changing_old_payload(
|
||
organizations,
|
||
):
|
||
seed(organizations[0])
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_user())
|
||
obj = OrganizationSourceRecord.objects.get()
|
||
url = f"/api/v2/organization-source-records/{obj.uid}/"
|
||
old = client.get(url)
|
||
assert old.status_code == 200, old.data
|
||
assert "lease_details" not in old.data.get("data", old.data)["payload"]
|
||
run_enrichment(lambda driver, url: page(), max_pages=1)
|
||
detail = client.get(url)
|
||
listing = client.get(
|
||
"/api/v2/organization-source-records/", {"source": "checko_leasing"}
|
||
)
|
||
assert detail.status_code == listing.status_code == 200
|
||
for body in [detail.data.get("data", detail.data), listing.data["data"][0]]:
|
||
assert body["payload"]["lease_details"]["contract_number"] == "TEST-42"
|
||
assert len(body["payload"]["lease_details"]["subjects"]) == 2
|
||
|
||
|
||
def test_refresh_api_details_mode_dispatches_without_loading_checko(organizations):
|
||
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
|
||
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_user(is_staff=True))
|
||
with patch.object(parse_checko_leasing, "apply_async") as dispatch:
|
||
response = client.post(
|
||
"/api/v1/sources/checko-leasing/refresh/",
|
||
{"params": {"mode": "details"}},
|
||
format="json",
|
||
)
|
||
assert response.status_code == 202, response.data
|
||
assert dispatch.call_args.kwargs["kwargs"]["mode"] == "details"
|
||
|
||
|
||
def test_refresh_rejects_invalid_mode_before_dispatch(organizations):
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_user(is_staff=True))
|
||
response = client.post(
|
||
"/api/v1/sources/checko-leasing/refresh/",
|
||
{"params": {"mode": "unsupported"}},
|
||
format="json",
|
||
)
|
||
assert response.status_code == 400
|
||
|
||
|
||
def test_detail_job_does_not_change_collection_success_or_use_checko(organizations):
|
||
from apps.parsers.models import ParserLoadLog
|
||
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
|
||
|
||
seed(organizations[0])
|
||
m = module()
|
||
with patch.object(
|
||
m, "public_browser", return_value=nullcontext(Mock())
|
||
), patch.object(m, "fetch_leasing_html", return_value=page()), patch(
|
||
"apps.parsers.checko_section_snapshots._client",
|
||
side_effect=AssertionError("No Checko calls in details mode"),
|
||
):
|
||
result = parse_checko_leasing.apply(kwargs={"mode": "details"}).get()
|
||
assert result["records_updated"] == 1
|
||
assert result["mode"] == "details"
|
||
assert ParserLoadLog.objects.count() == 0
|
||
|
||
|
||
def test_next_collection_keeps_details_for_same_event(organizations):
|
||
from tempfile import TemporaryFile
|
||
|
||
from apps.parsers.checko_section_snapshots import _collect_organization
|
||
from apps.parsers.models import ParserSourceArtifact
|
||
|
||
seed(organizations[0])
|
||
run_enrichment(lambda driver, url: page(), max_pages=1)
|
||
existing = OrganizationSourceRecord.objects.get()
|
||
fresh = SourceRecordInput(
|
||
external_id=existing.external_id,
|
||
title="Договор",
|
||
organization_name=organizations[0].name,
|
||
record_type="leasing_contract",
|
||
inn=organizations[0].inn,
|
||
ogrn=organizations[0].ogrn,
|
||
payload={"message_guid": GUID},
|
||
)
|
||
artifact = ParserSourceArtifact.objects.create(source="checko_leasing")
|
||
with TemporaryFile() as raw, patch(
|
||
"apps.parsers.checko_section_snapshots.collect_checko_section",
|
||
return_value=iter([fresh]),
|
||
):
|
||
_collect_organization(
|
||
None, organizations[0], "checko_leasing", artifact, 0, raw
|
||
)
|
||
staged = artifact.staged_records.get()
|
||
assert (
|
||
staged.normalized_data["payload"]["lease_details"]["contract_number"]
|
||
== "TEST-42"
|
||
)
|
||
|
||
|
||
def test_parallel_enrichment_is_rejected_before_browser_start(organizations):
|
||
seed(organizations[0])
|
||
cache.set(module().LOCK_KEY, "other-owner", timeout=3600)
|
||
result = run_enrichment(
|
||
lambda driver, url: pytest.fail("Second collector must not navigate"),
|
||
max_pages=1,
|
||
)
|
||
assert result["pages_requested"] == 0
|
||
assert result["error_code"] == "enrichment_already_running"
|
||
|
||
|
||
def test_successful_empty_run_does_not_open_browser(organizations):
|
||
m = module()
|
||
with patch.object(
|
||
m, "public_browser", side_effect=AssertionError("No browser for empty queue")
|
||
):
|
||
assert m.enrich_leasing_details()["pages_requested"] == 0
|
||
|
||
|
||
def test_targeted_run_deduplicates_compact_and_dashed_guid(organizations):
|
||
seed(organizations[0])
|
||
seed(organizations[1], GUID.replace("-", "").upper())
|
||
target = OrganizationSourceRecord.objects.get(
|
||
extension__organization=organizations[0]
|
||
)
|
||
result = run_enrichment(
|
||
lambda driver, url: page(), max_pages=1, record_uid=str(target.uid)
|
||
)
|
||
assert result["pages_requested"] == 1
|
||
assert result["records_updated"] == 2
|
||
|
||
|
||
def test_browser_startup_failure_has_explicit_result_and_cooldown(organizations):
|
||
from selenium.common.exceptions import WebDriverException
|
||
|
||
seed(organizations[0])
|
||
m = module()
|
||
with patch.object(
|
||
m, "public_browser", side_effect=WebDriverException("startup failed")
|
||
):
|
||
result = m.enrich_leasing_details(max_pages=1)
|
||
assert result["error_code"] == "browser_failure"
|
||
assert result["pages_requested"] == 0
|
||
assert cache.get(m.COOLDOWN_KEY)
|
||
assert not cache.get(m.LOCK_KEY)
|
||
|
||
|
||
def test_non_admin_cannot_launch_details(organizations):
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_user())
|
||
response = client.post(
|
||
"/api/v1/sources/checko-leasing/refresh/",
|
||
{"params": {"mode": "details"}},
|
||
format="json",
|
||
)
|
||
assert response.status_code == 403
|