feat: enrich saved leasing events with bounded Fedresurs details
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s
This commit is contained in:
@@ -149,6 +149,16 @@ def _canonical_records():
|
||||
else client_with(message_page, fixture("messages-page-2.json"))
|
||||
)
|
||||
row = next(collect_checko_section(client, organization=fssp_org, source=source))
|
||||
if source == "checko_leasing":
|
||||
from apps.parsers.leasing_details import parse_leasing_html
|
||||
|
||||
from tests.apps.parsers.test_leasing_details import URL, page
|
||||
|
||||
row.payload["lease_details"] = parse_leasing_html(page(), url=URL)
|
||||
row.payload["lease_details_status"] = {
|
||||
"status": "loaded",
|
||||
"checked_at": timezone.now().isoformat(),
|
||||
}
|
||||
row.payload["upstream"]["future_nested"] = {
|
||||
"keep": [None, False, 0, {"unknown": "0007"}]
|
||||
}
|
||||
|
||||
427
tests/apps/parsers/test_leasing_details.py
Normal file
427
tests/apps/parsers/test_leasing_details.py
Normal file
@@ -0,0 +1,427 @@
|
||||
"""Lease enrichment: real parsing/persistence, synthetic public-page transport."""
|
||||
|
||||
import importlib
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from apps.parsers.fedresurs_company_profile import FedresursUpstreamError
|
||||
from django.core.cache import cache
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from organizations.source_ingestion import (
|
||||
OrganizationSourceIngestionService,
|
||||
SourceRecordInput,
|
||||
)
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
GUID = "12345678-1234-4234-8234-123456789abc"
|
||||
URL = f"https://fedresurs.ru/sfactmessages/{GUID}"
|
||||
|
||||
|
||||
def page(kind="stop-financial-lease-contract-2", count=2):
|
||||
"""Synthetic DOM based on the public message components inspected on 06.10."""
|
||||
return f"""<sfact-message-card>
|
||||
<sfact-message-card-header><div class="header-item">12345678 от 17.09.2026 08:29:10 МСК</div></sfact-message-card-header>
|
||||
<sfact-message-{kind}>
|
||||
<div><div><span>Договор</span></div><div><span>TEST-42</span><span> от 08.12.2025</span></div></div>
|
||||
<div><div>Срок финансовой аренды</div><div>23.12.2025 - 15.09.2026</div></div>
|
||||
<div><div>Дата прекращения</div><div>15.09.2026</div></div>
|
||||
<div><div>Причина прекращения</div><div>Окончание договора</div></div>
|
||||
<div><div>Лизингодатели</div><div><message-participant><company-publisher>
|
||||
<div class="name">АО Тест 1</div><div><div>ИНН</div><div>7707083893</div></div>
|
||||
<div><div>ОГРН</div><div>1027700132195</div></div>
|
||||
</company-publisher></message-participant></div></div>
|
||||
<div><div>Лизингополучатели</div><div><message-participant><company-publisher>
|
||||
<div class="name">АО Тест 2</div><div><div>ИНН</div><div>7702070139</div></div>
|
||||
<div><div>ОГРН</div><div>1027700070518</div></div>
|
||||
</company-publisher></message-participant></div></div>
|
||||
<div>Предметы финансовой аренды (лизинга) ({count})</div>
|
||||
<sfact-message-subject-contract-table-2><table><thead><tr><th></th>
|
||||
<th>Предмет финансовой аренды (лизинга)</th><th>Описание</th></tr></thead><tbody>
|
||||
<tr><td>1</td><td><div><div>Идентификатор</div><div>нет данных</div></div>
|
||||
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
|
||||
<tr><td>2</td><td><div><div>Идентификатор</div><div>OBJ-TEST</div></div>
|
||||
<div><div>Классификатор</div><div>0104, Оборудование</div></div></td><td>Станок с ЧПУ</td></tr>
|
||||
</tbody></table></sfact-message-subject-contract-table-2>
|
||||
</sfact-message-{kind}>
|
||||
<sfact-message-card-msgs><a href="/sfactmessage/aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa">Заключение договора</a></sfact-message-card-msgs>
|
||||
</sfact-message-card>"""
|
||||
|
||||
|
||||
def module():
|
||||
try:
|
||||
return importlib.import_module("apps.parsers.leasing_details")
|
||||
except ImportError:
|
||||
pytest.fail("Lease detail enrichment is not implemented")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kind",
|
||||
[
|
||||
"financial-lease-contract-2",
|
||||
"change-financial-lease-contract-2",
|
||||
"stop-financial-lease-contract-2",
|
||||
"financial-lease-contract",
|
||||
],
|
||||
)
|
||||
def test_public_message_keeps_contract_parties_and_each_subject(kind):
|
||||
result = module().parse_leasing_html(page(kind), url=URL)
|
||||
assert result["contract_number"] == "TEST-42"
|
||||
assert result["contract_date"] == "2025-12-08"
|
||||
assert result["lease_start_date"] == "2025-12-23"
|
||||
assert result["lease_end_date"] == "2026-09-15"
|
||||
assert result["lessors"] == [
|
||||
{"name": "АО Тест 1", "inn": "7707083893", "ogrn": "1027700132195"}
|
||||
]
|
||||
assert result["lessees"][0]["inn"] == "7702070139"
|
||||
assert len(result["subjects"]) == 2
|
||||
assert result["subjects"][0]["identifier"] is None
|
||||
assert result["subjects"][1]["identifier"] == "OBJ-TEST"
|
||||
assert result["subjects"][0]["description"] == "Станок с ЧПУ"
|
||||
assert result["termination_reason"] == "Окончание договора"
|
||||
assert (
|
||||
result["related_messages"][0]["guid"] == "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url",
|
||||
[
|
||||
"http://fedresurs.ru/sfactmessages/" + GUID,
|
||||
"https://fedresurs.ru.evil.test/sfactmessages/" + GUID,
|
||||
"https://user@fedresurs.ru/sfactmessages/" + GUID,
|
||||
"https://fedresurs.ru/companies/" + GUID,
|
||||
],
|
||||
)
|
||||
def test_only_exact_public_message_urls_are_accepted(url):
|
||||
with pytest.raises(FedresursUpstreamError, match="invalid_leasing_message_url"):
|
||||
module().parse_leasing_html(page(), url=url)
|
||||
|
||||
|
||||
def test_partial_subject_table_is_not_published_as_complete():
|
||||
with pytest.raises(FedresursUpstreamError, match="incomplete_leasing_subjects"):
|
||||
module().parse_leasing_html(page(count=21), url=URL)
|
||||
|
||||
|
||||
def test_403_is_a_source_error_not_an_empty_contract():
|
||||
with pytest.raises(FedresursUpstreamError, match="upstream_access_blocked"):
|
||||
module().parse_leasing_html("<html><body>403 Forbidden</body></html>", url=URL)
|
||||
|
||||
|
||||
def test_browser_receives_explicit_user_agent_and_closes():
|
||||
from unittest.mock import Mock
|
||||
|
||||
from apps.parsers.fedresurs_browser import public_browser
|
||||
|
||||
driver = Mock()
|
||||
with patch(
|
||||
"apps.parsers.fedresurs_browser.webdriver.Chrome", return_value=driver
|
||||
) as chrome, public_browser(user_agent="Mozilla/5.0 ECOS-LeasingCollector/1.0"):
|
||||
assert (
|
||||
"--user-agent=Mozilla/5.0 ECOS-LeasingCollector/1.0"
|
||||
in chrome.call_args.kwargs["options"].arguments
|
||||
)
|
||||
driver.quit.assert_called_once()
|
||||
|
||||
|
||||
def seed(org, guid=GUID):
|
||||
return OrganizationSourceIngestionService.save_records(
|
||||
source="checko_leasing",
|
||||
load_batch=1,
|
||||
records=[
|
||||
SourceRecordInput(
|
||||
external_id=f"{org.inn}:{guid}",
|
||||
title="Заключение договора лизинга",
|
||||
organization_uid=org.uid,
|
||||
organization_name=org.name,
|
||||
inn=org.inn,
|
||||
ogrn=org.ogrn,
|
||||
record_type="leasing_contract",
|
||||
status="published",
|
||||
url=f"https://fedresurs.ru/sfactmessages/{guid}",
|
||||
payload={
|
||||
"message_guid": guid,
|
||||
"message_type": "FinancialLeaseContract2",
|
||||
"lease_event_type": "FinancialLeaseContract2",
|
||||
"message_type_name": "Заключение договора лизинга",
|
||||
"publication_raw": "2026-09-17",
|
||||
"publication_date": "2026-09-17",
|
||||
"publisher_identifier": org.ogrn,
|
||||
"participant_identifiers": [org.ogrn],
|
||||
"source_url": f"https://fedresurs.ru/sfactmessages/{guid}",
|
||||
"fetched_at": timezone.now().isoformat(),
|
||||
"upstream": {"GUID": guid},
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def organizations(db):
|
||||
cache.clear()
|
||||
return [
|
||||
Organization.objects.create(
|
||||
name=f"АО Тест {n}",
|
||||
inn=inn,
|
||||
ogrn=ogrn,
|
||||
okpo=f"1234567{n}",
|
||||
directory_imported_at=timezone.now(),
|
||||
opk_registry_membership=True,
|
||||
)
|
||||
for n, (inn, ogrn) in enumerate(
|
||||
[("7707083893", "1027700132195"), ("7702070139", "1027700070518")]
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def run_enrichment(fetch, **kwargs):
|
||||
m = module()
|
||||
with patch.object(
|
||||
m, "public_browser", return_value=nullcontext(Mock())
|
||||
), patch.object(m, "fetch_leasing_html", side_effect=fetch):
|
||||
return m.enrich_leasing_details(**kwargs)
|
||||
|
||||
|
||||
def test_one_page_per_guid_and_second_run_uses_persisted_details(organizations):
|
||||
for org in organizations:
|
||||
seed(org)
|
||||
before = list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
|
||||
visits = []
|
||||
|
||||
def fetch(driver, url):
|
||||
visits.append(url)
|
||||
return page()
|
||||
|
||||
first = run_enrichment(fetch, max_pages=1)
|
||||
second = run_enrichment(fetch, max_pages=1)
|
||||
assert first["pages_requested"] == 1
|
||||
assert first["records_updated"] == 2
|
||||
assert second["pages_requested"] == 0
|
||||
assert len(visits) == 1
|
||||
assert (
|
||||
list(OrganizationSourceRecord.objects.values_list("uid", "load_batch"))
|
||||
== before
|
||||
)
|
||||
for obj in OrganizationSourceRecord.objects.all():
|
||||
assert obj.payload["lease_details"]["subjects"][1]["identifier"] == "OBJ-TEST"
|
||||
|
||||
|
||||
def test_page_budget_is_enforced_and_next_run_advances(organizations):
|
||||
seed(organizations[0])
|
||||
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||||
first = run_enrichment(lambda driver, url: page(), max_pages=1)
|
||||
assert first["pages_requested"] == 1
|
||||
assert (
|
||||
OrganizationSourceRecord.objects.filter(
|
||||
payload__lease_details_status__status="loaded"
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
second = run_enrichment(lambda driver, url: page(), max_pages=1)
|
||||
assert second["pages_requested"] == 1
|
||||
assert (
|
||||
OrganizationSourceRecord.objects.filter(
|
||||
payload__lease_details_status__status="loaded"
|
||||
).count()
|
||||
== 2
|
||||
)
|
||||
|
||||
|
||||
def test_block_stops_batch_and_cooldown_prevents_next_batch(organizations):
|
||||
seed(organizations[0])
|
||||
seed(organizations[0], "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
|
||||
|
||||
def blocked(driver, url):
|
||||
raise FedresursUpstreamError("upstream_access_blocked")
|
||||
|
||||
result = run_enrichment(blocked, max_pages=2)
|
||||
assert result["pages_requested"] == 1
|
||||
assert result["error_code"] == "upstream_access_blocked"
|
||||
assert (
|
||||
OrganizationSourceRecord.objects.filter(
|
||||
payload__lease_details_status__status="failed"
|
||||
).count()
|
||||
== 1
|
||||
)
|
||||
again = run_enrichment(
|
||||
lambda driver, url: pytest.fail("Cooldown must prevent network access"),
|
||||
max_pages=2,
|
||||
)
|
||||
assert again["pages_requested"] == 0
|
||||
assert again["error_code"] == "upstream_cooldown"
|
||||
|
||||
|
||||
def test_subject_identity_mismatch_never_enriches_another_organization(organizations):
|
||||
seed(organizations[0])
|
||||
result = run_enrichment(
|
||||
lambda driver, url: page()
|
||||
.replace("7707083893", "9999999999")
|
||||
.replace("1027700132195", "1999999999999"),
|
||||
max_pages=1,
|
||||
)
|
||||
obj = OrganizationSourceRecord.objects.get()
|
||||
assert "lease_details" not in obj.payload
|
||||
assert result["records_updated"] == 0
|
||||
assert result["error_code"] == "leasing_party_mismatch"
|
||||
assert obj.payload["lease_details_status"]["error_code"] == "leasing_party_mismatch"
|
||||
|
||||
|
||||
def test_list_and_detail_expose_enriched_subjects_without_changing_old_payload(
|
||||
organizations,
|
||||
):
|
||||
seed(organizations[0])
|
||||
client = APIClient()
|
||||
client.force_authenticate(UserFactory.create_user())
|
||||
obj = OrganizationSourceRecord.objects.get()
|
||||
url = f"/api/v2/organization-source-records/{obj.uid}/"
|
||||
old = client.get(url)
|
||||
assert old.status_code == 200, old.data
|
||||
assert "lease_details" not in old.data.get("data", old.data)["payload"]
|
||||
run_enrichment(lambda driver, url: page(), max_pages=1)
|
||||
detail = client.get(url)
|
||||
listing = client.get(
|
||||
"/api/v2/organization-source-records/", {"source": "checko_leasing"}
|
||||
)
|
||||
assert detail.status_code == listing.status_code == 200
|
||||
for body in [detail.data.get("data", detail.data), listing.data["data"][0]]:
|
||||
assert body["payload"]["lease_details"]["contract_number"] == "TEST-42"
|
||||
assert len(body["payload"]["lease_details"]["subjects"]) == 2
|
||||
|
||||
|
||||
def test_refresh_api_details_mode_dispatches_without_loading_checko(organizations):
|
||||
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
|
||||
|
||||
client = APIClient()
|
||||
client.force_authenticate(UserFactory.create_user(is_staff=True))
|
||||
with patch.object(parse_checko_leasing, "apply_async") as dispatch:
|
||||
response = client.post(
|
||||
"/api/v1/sources/checko-leasing/refresh/",
|
||||
{"params": {"mode": "details"}},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 202, response.data
|
||||
assert dispatch.call_args.kwargs["kwargs"]["mode"] == "details"
|
||||
|
||||
|
||||
def test_refresh_rejects_invalid_mode_before_dispatch(organizations):
|
||||
client = APIClient()
|
||||
client.force_authenticate(UserFactory.create_user(is_staff=True))
|
||||
response = client.post(
|
||||
"/api/v1/sources/checko-leasing/refresh/",
|
||||
{"params": {"mode": "unsupported"}},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
def test_detail_job_does_not_change_collection_success_or_use_checko(organizations):
|
||||
from apps.parsers.models import ParserLoadLog
|
||||
from apps.parsers.tasks_registry_snapshots import parse_checko_leasing
|
||||
|
||||
seed(organizations[0])
|
||||
m = module()
|
||||
with patch.object(
|
||||
m, "public_browser", return_value=nullcontext(Mock())
|
||||
), patch.object(m, "fetch_leasing_html", return_value=page()), patch(
|
||||
"apps.parsers.checko_section_snapshots._client",
|
||||
side_effect=AssertionError("No Checko calls in details mode"),
|
||||
):
|
||||
result = parse_checko_leasing.apply(kwargs={"mode": "details"}).get()
|
||||
assert result["records_updated"] == 1
|
||||
assert result["mode"] == "details"
|
||||
assert ParserLoadLog.objects.count() == 0
|
||||
|
||||
|
||||
def test_next_collection_keeps_details_for_same_event(organizations):
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
from apps.parsers.checko_section_snapshots import _collect_organization
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
|
||||
seed(organizations[0])
|
||||
run_enrichment(lambda driver, url: page(), max_pages=1)
|
||||
existing = OrganizationSourceRecord.objects.get()
|
||||
fresh = SourceRecordInput(
|
||||
external_id=existing.external_id,
|
||||
title="Договор",
|
||||
organization_name=organizations[0].name,
|
||||
record_type="leasing_contract",
|
||||
inn=organizations[0].inn,
|
||||
ogrn=organizations[0].ogrn,
|
||||
payload={"message_guid": GUID},
|
||||
)
|
||||
artifact = ParserSourceArtifact.objects.create(source="checko_leasing")
|
||||
with TemporaryFile() as raw, patch(
|
||||
"apps.parsers.checko_section_snapshots.collect_checko_section",
|
||||
return_value=iter([fresh]),
|
||||
):
|
||||
_collect_organization(
|
||||
None, organizations[0], "checko_leasing", artifact, 0, raw
|
||||
)
|
||||
staged = artifact.staged_records.get()
|
||||
assert (
|
||||
staged.normalized_data["payload"]["lease_details"]["contract_number"]
|
||||
== "TEST-42"
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_enrichment_is_rejected_before_browser_start(organizations):
|
||||
seed(organizations[0])
|
||||
cache.set(module().LOCK_KEY, "other-owner", timeout=3600)
|
||||
result = run_enrichment(
|
||||
lambda driver, url: pytest.fail("Second collector must not navigate"),
|
||||
max_pages=1,
|
||||
)
|
||||
assert result["pages_requested"] == 0
|
||||
assert result["error_code"] == "enrichment_already_running"
|
||||
|
||||
|
||||
def test_successful_empty_run_does_not_open_browser(organizations):
|
||||
m = module()
|
||||
with patch.object(
|
||||
m, "public_browser", side_effect=AssertionError("No browser for empty queue")
|
||||
):
|
||||
assert m.enrich_leasing_details()["pages_requested"] == 0
|
||||
|
||||
|
||||
def test_targeted_run_deduplicates_compact_and_dashed_guid(organizations):
|
||||
seed(organizations[0])
|
||||
seed(organizations[1], GUID.replace("-", "").upper())
|
||||
target = OrganizationSourceRecord.objects.get(
|
||||
extension__organization=organizations[0]
|
||||
)
|
||||
result = run_enrichment(
|
||||
lambda driver, url: page(), max_pages=1, record_uid=str(target.uid)
|
||||
)
|
||||
assert result["pages_requested"] == 1
|
||||
assert result["records_updated"] == 2
|
||||
|
||||
|
||||
def test_browser_startup_failure_has_explicit_result_and_cooldown(organizations):
|
||||
from selenium.common.exceptions import WebDriverException
|
||||
|
||||
seed(organizations[0])
|
||||
m = module()
|
||||
with patch.object(
|
||||
m, "public_browser", side_effect=WebDriverException("startup failed")
|
||||
):
|
||||
result = m.enrich_leasing_details(max_pages=1)
|
||||
assert result["error_code"] == "browser_failure"
|
||||
assert result["pages_requested"] == 0
|
||||
assert cache.get(m.COOLDOWN_KEY)
|
||||
assert not cache.get(m.LOCK_KEY)
|
||||
|
||||
|
||||
def test_non_admin_cannot_launch_details(organizations):
|
||||
client = APIClient()
|
||||
client.force_authenticate(UserFactory.create_user())
|
||||
response = client.post(
|
||||
"/api/v1/sources/checko-leasing/refresh/",
|
||||
{"params": {"mode": "details"}},
|
||||
format="json",
|
||||
)
|
||||
assert response.status_code == 403
|
||||
Reference in New Issue
Block a user