feat: enrich saved leasing events with bounded Fedresurs details
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s

This commit is contained in:
Aleksandr Meshchryakov
2026-10-06 14:43:21 +02:00
parent 897941583e
commit ccb7f45c7e
15 changed files with 1418 additions and 4 deletions

View File

@@ -149,6 +149,16 @@ def _canonical_records():
else client_with(message_page, fixture("messages-page-2.json"))
)
row = next(collect_checko_section(client, organization=fssp_org, source=source))
if source == "checko_leasing":
from apps.parsers.leasing_details import parse_leasing_html
from tests.apps.parsers.test_leasing_details import URL, page
row.payload["lease_details"] = parse_leasing_html(page(), url=URL)
row.payload["lease_details_status"] = {
"status": "loaded",
"checked_at": timezone.now().isoformat(),
}
row.payload["upstream"]["future_nested"] = {
"keep": [None, False, 0, {"unknown": "0007"}]
}

View File

@@ -0,0 +1,427 @@
"""Lease enrichment: real parsing/persistence, synthetic public-page transport."""
import importlib
from contextlib import nullcontext
from unittest.mock import Mock, patch
import pytest
from apps.parsers.fedresurs_company_profile import FedresursUpstreamError
from django.core.cache import cache
from django.utils import timezone
from organizations.models import Organization, OrganizationSourceRecord
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
GUID = "12345678-1234-4234-8234-123456789abc"
URL = f"https://fedresurs.ru/sfactmessages/{GUID}"
def page(kind="stop-financial-lease-contract-2", count=2):
"""Synthetic DOM based on the public message components inspected on 06.10."""
return f"""<sfact-message-card>
<sfact-message-card-header><div class="header-item">12345678 от 17.09.2026 08:29:10 МСК</div></sfact-message-card-header>
<sfact-message-{kind}>
<div><div><span>Договор</span></div><div><span>TEST-42</span><span> от 08.12.2025</span></div></div>
<div><div>Срок финансовой аренды</div><div>23.12.2025 - 15.09.2026</div></div>
<div><div>Дата прекращения</div><div>15.09.2026</div></div>
<div><div>Причина прекращения</div><div>Окончание договора</div></div>
<div><div>Лизингодатели</div><div><message-participant><company-publisher>
<div class="name">АО Тест 1</div><div><div>ИНН</div><div>7707083893</div></div>
<div><div>ОГРН</div><div>1027700132195</div></div>
</company-publisher></message-participant></div></div>
<div><div>Лизингополучатели</div><div><message-participant><company-publisher>
<div class="name">АО Тест 2</div><div><div>ИНН</div><div>7702070139</div></div>
<div><div>ОГРН</div><div>1027700070518</div></div>
</company-publisher></message-participant></div></div>
<div>Предметы финансовой аренды (лизинга) ({count})</div>
<sfact-message-subject-contract-table-2><table><thead><tr><th></th>
<th>Предмет финансовой аренды (лизинга)</th><th>Описание</th></tr></thead><tbody>
<tr><td>1</td><td><div><div>Идентификатор</div><div>нет данных</div></div>
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
<tr><td>2</td><td><div><div>Идентификатор</div><div>OBJ-TEST</div></div>
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
</tbody></table></sfact-message-subject-contract-table-2>
</sfact-message-{kind}>
<sfact-message-card-msgs><a href="/sfactmessage/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa">Заключение договора</a></sfact-message-card-msgs>
</sfact-message-card>"""
def module():
try:
return importlib.import_module("apps.parsers.leasing_details")
except ImportError:
pytest.fail("Lease detail enrichment is not implemented")
@pytest.mark.parametrize(
"kind",
[
"financial-lease-contract-2",
"change-financial-lease-contract-2",
"stop-financial-lease-contract-2",
"financial-lease-contract",
],
)
def test_public_message_keeps_contract_parties_and_each_subject(kind):
result = module().parse_leasing_html(page(kind), url=URL)
assert result["contract_number"] == "TEST-42"
assert result["contract_date"] == "2025-12-08"
assert result["lease_start_date"] == "2025-12-23"
assert result["lease_end_date"] == "2026-09-15"
assert result["lessors"] == [
{"name": "АО Тест 1", "inn": "7707083893", "ogrn": "1027700132195"}
]
assert result["lessees"][0]["inn"] == "7702070139"
assert len(result["subjects"]) == 2
assert result["subjects"][0]["identifier"] is None
assert result["subjects"][1]["identifier"] == "OBJ-TEST"
assert result["subjects"][0]["description"] == "Станок с ЧПУ"
assert result["termination_reason"] == "Окончание договора"
assert (
result["related_messages"][0]["guid"] == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
)
@pytest.mark.parametrize(
"url",
[
"http://fedresurs.ru/sfactmessages/" + GUID,
"https://fedresurs.ru.evil.test/sfactmessages/" + GUID,
"https://user@fedresurs.ru/sfactmessages/" + GUID,
"https://fedresurs.ru/companies/" + GUID,
],
)
def test_only_exact_public_message_urls_are_accepted(url):
with pytest.raises(FedresursUpstreamError, match="invalid_leasing_message_url"):
module().parse_leasing_html(page(), url=url)
def test_partial_subject_table_is_not_published_as_complete():
with pytest.raises(FedresursUpstreamError, match="incomplete_leasing_subjects"):
module().parse_leasing_html(page(count=21), url=URL)
def test_403_is_a_source_error_not_an_empty_contract():
with pytest.raises(FedresursUpstreamError, match="upstream_access_blocked"):
module().parse_leasing_html("<html><body>403 Forbidden</body></html>", url=URL)
def test_browser_receives_explicit_user_agent_and_closes():
from unittest.mock import Mock
from apps.parsers.fedresurs_browser import public_browser
driver = Mock()
with patch(
"apps.parsers.fedresurs_browser.webdriver.Chrome", return_value=driver
) as chrome, public_browser(user_agent="Mozilla/5.0 ECOS-LeasingCollector/1.0"):
assert (
"--user-agent=Mozilla/5.0 ECOS-LeasingCollector/1.0"
in chrome.call_args.kwargs["options"].arguments
)
driver.quit.assert_called_once()
def seed(org, guid=GUID):
return OrganizationSourceIngestionService.save_records(
source="checko_leasing",
load_batch=1,
records=[
SourceRecordInput(
external_id=f"{org.inn}:{guid}",
title="Заключение договора лизинга",
organization_uid=org.uid,
organization_name=org.name,
inn=org.inn,
ogrn=org.ogrn,
record_type="leasing_contract",
status="published",
url=f"https://fedresurs.ru/sfactmessages/{guid}",
payload={
"message_guid": guid,
"message_type": "FinancialLeaseContract2",
"lease_event_type": "FinancialLeaseContract2",
"message_type_name": "Заключение договора лизинга",
"publication_raw": "2026-09-17",
"publication_date": "2026-09-17",
"publisher_identifier": org.ogrn,
"participant_identifiers": [org.ogrn],
"source_url": f"https://fedresurs.ru/sfactmessages/{guid}",
"fetched_at": timezone.now().isoformat(),
"upstream": {"GUID": guid},
},
)
],
)
@pytest.fixture
def organizations(db):
cache.clear()
return [
Organization.objects.create(
name=f"АО Тест {n}",
inn=inn,
ogrn=ogrn,
okpo=f"1234567{n}",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
for n, (inn, ogrn) in enumerate(
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
)
]
def run_enrichment(fetch, **kwargs):
m = module()
with patch.object(
m, "public_browser", return_value=nullcontext(Mock())
), patch.object(m, "fetch_leasing_html", side_effect=fetch):
return m.enrich_leasing_details(**kwargs)
def test_one_page_per_guid_and_second_run_uses_persisted_details(organizations):
for org in organizations:
seed(org)
before = list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
visits = []
def fetch(driver, url):
visits.append(url)
return page()
first = run_enrichment(fetch, max_pages=1)
second = run_enrichment(fetch, max_pages=1)
assert first["pages_requested"] == 1
assert first["records_updated"] == 2
assert second["pages_requested"] == 0
assert len(visits) == 1
assert (
list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
== before
)
for obj in OrganizationSourceRecord.objects.all():
assert obj.payload["lease_details"]["subjects"][1]["identifier"] == "OBJ-TEST"
def test_page_budget_is_enforced_and_next_run_advances(organizations):
seed(organizations[0])
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
first = run_enrichment(lambda driver, url: page(), max_pages=1)
assert first["pages_requested"] == 1
assert (
OrganizationSourceRecord.objects.filter(
payload__lease_details_status__status="loaded"
).count()
== 1
)
second = run_enrichment(lambda driver, url: page(), max_pages=1)
assert second["pages_requested"] == 1
assert (
OrganizationSourceRecord.objects.filter(
payload__lease_details_status__status="loaded"
).count()
== 2
)
def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations):
seed(organizations[0])
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
def blocked(driver, url):
raise FedresursUpstreamError("upstream_access_blocked")
result = run_enrichment(blocked, max_pages=2)
assert result["pages_requested"] == 1
assert result["error_code"] == "upstream_access_blocked"
assert (
OrganizationSourceRecord.objects.filter(
payload__lease_details_status__status="failed"
).count()
== 1
)
again = run_enrichment(
lambda driver, url: pytest.fail("Cooldown must prevent network access"),
max_pages=2,
)
assert again["pages_requested"] == 0
assert again["error_code"] == "upstream_cooldown"
def test_subject_identity_mismatch_never_enriches_another_organization(organizations):
seed(organizations[0])
result = run_enrichment(
lambda driver, url: page()
.replace("7707083893", "9999999999")
.replace("1027700132195", "1999999999999"),
max_pages=1,
)
obj = OrganizationSourceRecord.objects.get()
assert "lease_details" not in obj.payload
assert result["records_updated"] == 0
assert result["error_code"] == "leasing_party_mismatch"
assert obj.payload["lease_details_status"]["error_code"] == "leasing_party_mismatch"
def test_list_and_detail_expose_enriched_subjects_without_changing_old_payload(
organizations,
):
seed(organizations[0])
client = APIClient()
client.force_authenticate(UserFactory.create_user())
obj = OrganizationSourceRecord.objects.get()
url = f"/api/v2/organization-source-records/{obj.uid}/"
old = client.get(url)
assert old.status_code == 200, old.data
assert "lease_details" not in old.data.get("data", old.data)["payload"]
run_enrichment(lambda driver, url: page(), max_pages=1)
detail = client.get(url)
listing = client.get(
"/api/v2/organization-source-records/", {"source": "checko_leasing"}
)
assert detail.status_code == listing.status_code == 200
for body in [detail.data.get("data", detail.data), listing.data["data"][0]]:
assert body["payload"]["lease_details"]["contract_number"] == "TEST-42"
assert len(body["payload"]["lease_details"]["subjects"]) == 2
def test_refresh_api_details_mode_dispatches_without_loading_checko(organizations):
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
client = APIClient()
client.force_authenticate(UserFactory.create_user(is_staff=True))
with patch.object(parse_checko_leasing, "apply_async") as dispatch:
response = client.post(
"/api/v1/sources/checko-leasing/refresh/",
{"params": {"mode": "details"}},
format="json",
)
assert response.status_code == 202, response.data
assert dispatch.call_args.kwargs["kwargs"]["mode"] == "details"
def test_refresh_rejects_invalid_mode_before_dispatch(organizations):
client = APIClient()
client.force_authenticate(UserFactory.create_user(is_staff=True))
response = client.post(
"/api/v1/sources/checko-leasing/refresh/",
{"params": {"mode": "unsupported"}},
format="json",
)
assert response.status_code == 400
def test_detail_job_does_not_change_collection_success_or_use_checko(organizations):
from apps.parsers.models import ParserLoadLog
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
seed(organizations[0])
m = module()
with patch.object(
m, "public_browser", return_value=nullcontext(Mock())
), patch.object(m, "fetch_leasing_html", return_value=page()), patch(
"apps.parsers.checko_section_snapshots._client",
side_effect=AssertionError("No Checko calls in details mode"),
):
result = parse_checko_leasing.apply(kwargs={"mode": "details"}).get()
assert result["records_updated"] == 1
assert result["mode"] == "details"
assert ParserLoadLog.objects.count() == 0
def test_next_collection_keeps_details_for_same_event(organizations):
from tempfile import TemporaryFile
from apps.parsers.checko_section_snapshots import _collect_organization
from apps.parsers.models import ParserSourceArtifact
seed(organizations[0])
run_enrichment(lambda driver, url: page(), max_pages=1)
existing = OrganizationSourceRecord.objects.get()
fresh = SourceRecordInput(
external_id=existing.external_id,
title="Договор",
organization_name=organizations[0].name,
record_type="leasing_contract",
inn=organizations[0].inn,
ogrn=organizations[0].ogrn,
payload={"message_guid": GUID},
)
artifact = ParserSourceArtifact.objects.create(source="checko_leasing")
with TemporaryFile() as raw, patch(
"apps.parsers.checko_section_snapshots.collect_checko_section",
return_value=iter([fresh]),
):
_collect_organization(
None, organizations[0], "checko_leasing", artifact, 0, raw
)
staged = artifact.staged_records.get()
assert (
staged.normalized_data["payload"]["lease_details"]["contract_number"]
== "TEST-42"
)
def test_parallel_enrichment_is_rejected_before_browser_start(organizations):
seed(organizations[0])
cache.set(module().LOCK_KEY, "other-owner", timeout=3600)
result = run_enrichment(
lambda driver, url: pytest.fail("Second collector must not navigate"),
max_pages=1,
)
assert result["pages_requested"] == 0
assert result["error_code"] == "enrichment_already_running"
def test_successful_empty_run_does_not_open_browser(organizations):
m = module()
with patch.object(
m, "public_browser", side_effect=AssertionError("No browser for empty queue")
):
assert m.enrich_leasing_details()["pages_requested"] == 0
def test_targeted_run_deduplicates_compact_and_dashed_guid(organizations):
seed(organizations[0])
seed(organizations[1], GUID.replace("-", "").upper())
target = OrganizationSourceRecord.objects.get(
extension__organization=organizations[0]
)
result = run_enrichment(
lambda driver, url: page(), max_pages=1, record_uid=str(target.uid)
)
assert result["pages_requested"] == 1
assert result["records_updated"] == 2
def test_browser_startup_failure_has_explicit_result_and_cooldown(organizations):
from selenium.common.exceptions import WebDriverException
seed(organizations[0])
m = module()
with patch.object(
m, "public_browser", side_effect=WebDriverException("startup failed")
):
result = m.enrich_leasing_details(max_pages=1)
assert result["error_code"] == "browser_failure"
assert result["pages_requested"] == 0
assert cache.get(m.COOLDOWN_KEY)
assert not cache.get(m.LOCK_KEY)
def test_non_admin_cannot_launch_details(organizations):
client = APIClient()
client.force_authenticate(UserFactory.create_user())
response = client.post(
"/api/v1/sources/checko-leasing/refresh/",
{"params": {"mode": "details"}},
format="json",
)
assert response.status_code == 403