feat: enrich saved leasing events with bounded Fedresurs details
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m26s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m28s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 2m3s
This commit is contained in:
@@ -22,7 +22,7 @@ from apps.parsers.registry_snapshots import (
|
||||
)
|
||||
from django.conf import settings
|
||||
from django.db.models import Q
|
||||
from organizations.models import Organization
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
|
||||
|
||||
def _client():
|
||||
@@ -78,6 +78,16 @@ def _collect_organization(client, org, source, artifact, row_number, raw_file):
|
||||
raise SnapshotValidationError("checko_record_outside_claim_scope")
|
||||
count += 1
|
||||
row_number += 1
|
||||
if source == "checko_leasing":
|
||||
previous = OrganizationSourceRecord.objects.filter(
|
||||
source=source,
|
||||
extension__organization=org,
|
||||
external_id=row.external_id,
|
||||
).first()
|
||||
if previous is not None:
|
||||
for key in ("lease_details", "lease_details_status"):
|
||||
if key in previous.payload:
|
||||
row.payload[key] = previous.payload[key]
|
||||
normalized = {
|
||||
"title": row.title,
|
||||
"record_date": row.record_date,
|
||||
|
||||
@@ -34,13 +34,17 @@ def _wait_public_element(browser, by: str, value: str):
|
||||
|
||||
|
||||
@contextmanager
|
||||
def public_browser() -> Iterator[webdriver.Chrome]:
|
||||
def public_browser(*, user_agent: str | None = None) -> Iterator[webdriver.Chrome]:
|
||||
"""Start a fresh browser, with no inherited profile, cookies, or stealth flags."""
|
||||
options = webdriver.ChromeOptions()
|
||||
options.binary_location = settings.FEDRESURS_BROWSER_BINARY
|
||||
options.add_argument("--headless=new")
|
||||
options.add_argument("--no-sandbox")
|
||||
options.add_argument("--disable-dev-shm-usage")
|
||||
if user_agent is not None:
|
||||
if not user_agent or "\n" in user_agent or "\r" in user_agent:
|
||||
raise FedresursUpstreamError("invalid_browser_user_agent")
|
||||
options.add_argument(f"--user-agent={user_agent}")
|
||||
driver = webdriver.Chrome(
|
||||
service=Service(executable_path=settings.FEDRESURS_BROWSER_DRIVER),
|
||||
options=options,
|
||||
|
||||
@@ -32,7 +32,11 @@ SECTION_NAMES = {
|
||||
"trades": "biddings",
|
||||
}
|
||||
CHALLENGE_WORDS = ("captcha", "qrator", "access denied", "проверка браузера")
|
||||
ACCESS_BLOCK_WORDS = ("403 error", "access to fedresurs.ru is forbidden")
|
||||
ACCESS_BLOCK_WORDS = (
|
||||
"403 error",
|
||||
"403 forbidden",
|
||||
"access to fedresurs.ru is forbidden",
|
||||
)
|
||||
MAX_RENDERED_SECTION_BYTES = 4 * 1024 * 1024
|
||||
PUBLIC_PAGE_ROOTS = {
|
||||
"publications": "app-company-card-publications",
|
||||
|
||||
378
src/apps/parsers/leasing_details.py
Normal file
378
src/apps/parsers/leasing_details.py
Normal file
@@ -0,0 +1,378 @@
|
||||
"""Bounded enrichment of saved lease events from public Fedresurs message pages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from apps.parsers.fedresurs_browser import public_browser
|
||||
from apps.parsers.fedresurs_company_profile import (
|
||||
FEDRESURS_ORIGIN,
|
||||
FedresursUpstreamError,
|
||||
_challenge,
|
||||
_iso_date,
|
||||
_label_value,
|
||||
)
|
||||
from bs4 import BeautifulSoup
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db import transaction
|
||||
from django.utils import timezone
|
||||
from django.utils.dateparse import parse_datetime
|
||||
from organizations.cache import invalidate_organization_api_cache
|
||||
from organizations.models import OrganizationSourceRecord
|
||||
from selenium.common.exceptions import TimeoutException, WebDriverException
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
SOURCE = "checko_leasing"
|
||||
LOCK_KEY = "fedresurs:leasing:enrichment-lock"
|
||||
COOLDOWN_KEY = "fedresurs:leasing:enrichment-cooldown"
|
||||
LOCK_SECONDS = 3600
|
||||
SOURCE_ERRORS = frozenset(
|
||||
{
|
||||
"upstream_access_blocked",
|
||||
"upstream_challenge",
|
||||
"upstream_timeout",
|
||||
"browser_failure",
|
||||
}
|
||||
)
|
||||
CONTENT_TAG = re.compile(
|
||||
r"^sfact-message-(?:financial-lease-contract|change-financial-lease-contract|stop-financial-lease-contract)(?:-2)?$"
|
||||
)
|
||||
|
||||
|
||||
def message_guid(url: str) -> str:
|
||||
"""Accept only anonymous public message URLs, including compact GUIDs."""
|
||||
parsed = urlsplit(url)
|
||||
match = re.fullmatch(r"/sfactmessages?/([0-9a-fA-F-]{32,36})", parsed.path)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.netloc != "fedresurs.ru"
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or match is None
|
||||
):
|
||||
raise FedresursUpstreamError("invalid_leasing_message_url")
|
||||
try:
|
||||
return str(uuid.UUID(match.group(1)))
|
||||
except ValueError as exc:
|
||||
raise FedresursUpstreamError("invalid_leasing_message_url") from exc
|
||||
|
||||
|
||||
def _parties(content, label: str) -> list[dict]:
|
||||
for node in content.find_all(string=True):
|
||||
if str(node).strip() != label:
|
||||
continue
|
||||
parent = node.parent.parent
|
||||
return [
|
||||
{
|
||||
"name": participant.select_one(".name").get_text(" ", strip=True)
|
||||
if participant.select_one(".name") is not None
|
||||
else None,
|
||||
"inn": _label_value(participant, "ИНН") or None,
|
||||
"ogrn": _label_value(participant, "ОГРН") or None,
|
||||
}
|
||||
for participant in parent.find_all("message-participant")
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def _lease_value(content, label: str) -> str:
|
||||
"""Labels may contain a span; find their value without depending on CSS layout."""
|
||||
for node in content.find_all(string=True):
|
||||
if str(node).strip() != label:
|
||||
continue
|
||||
ancestor = node.parent
|
||||
while ancestor is not None and ancestor is not content:
|
||||
if ancestor.get_text(" ", strip=True) != label:
|
||||
break
|
||||
sibling = ancestor.find_next_sibling()
|
||||
if sibling is not None:
|
||||
return sibling.get_text(" ", strip=True)
|
||||
ancestor = ancestor.parent
|
||||
return ""
|
||||
|
||||
|
||||
def _subjects(content) -> list[dict]:
|
||||
subjects = []
|
||||
for table in content.find_all("table"):
|
||||
if "Предмет финансовой аренды" not in table.get_text(" ", strip=True):
|
||||
continue
|
||||
for row in table.select("tbody > tr"):
|
||||
cells = row.find_all("td", recursive=False)
|
||||
if len(cells) != 3:
|
||||
raise FedresursUpstreamError("incomplete_leasing_subjects")
|
||||
identifier_raw = _label_value(cells[1], "Идентификатор")
|
||||
classifier = _label_value(cells[1], "Классификатор")
|
||||
code, separator, name = classifier.partition(",")
|
||||
subjects.append(
|
||||
{
|
||||
"identifier": identifier_raw
|
||||
if identifier_raw and identifier_raw.casefold() != "нет данных"
|
||||
else None,
|
||||
"identifier_raw": identifier_raw or None,
|
||||
"classifier_code": code.strip() or None,
|
||||
"classifier_name": name.strip() if separator else None,
|
||||
"description": cells[2].get_text("\n", strip=True) or None,
|
||||
}
|
||||
)
|
||||
count = re.search(
|
||||
r"Предметы финансовой аренды\s*\(лизинга\)\s*\((\d+)\)",
|
||||
content.get_text(" ", strip=True),
|
||||
)
|
||||
if not subjects or (count is not None and int(count.group(1)) != len(subjects)):
|
||||
raise FedresursUpstreamError("incomplete_leasing_subjects")
|
||||
return subjects
|
||||
|
||||
|
||||
def parse_leasing_html(html: str, *, url: str) -> dict:
|
||||
message_guid(url)
|
||||
_challenge(html)
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
root = soup.find("sfact-message-card")
|
||||
content = root.find(CONTENT_TAG) if root else None
|
||||
if content is None:
|
||||
raise FedresursUpstreamError("incomplete_leasing_message")
|
||||
contract_raw = _lease_value(content, "Договор")
|
||||
contract = re.fullmatch(r"(.+?)\s+от\s+(\d{2}\.\d{2}\.\d{4})", contract_raw)
|
||||
dates = re.findall(
|
||||
r"\d{2}\.\d{2}\.\d{4}", _label_value(content, "Срок финансовой аренды")
|
||||
)
|
||||
related = {}
|
||||
for link in root.select("sfact-message-card-msgs a[href]"):
|
||||
target = urljoin(FEDRESURS_ORIGIN, link["href"])
|
||||
try:
|
||||
guid = message_guid(target)
|
||||
except FedresursUpstreamError:
|
||||
continue
|
||||
related[guid] = {
|
||||
"guid": guid,
|
||||
"url": target,
|
||||
"label": link.get_text(" ", strip=True),
|
||||
}
|
||||
if contract is None or len(dates) != 2:
|
||||
raise FedresursUpstreamError("incomplete_leasing_contract")
|
||||
result = {
|
||||
"contract_number": contract.group(1),
|
||||
"contract_date": _iso_date(contract.group(2)),
|
||||
"lease_start_date": _iso_date(dates[0]),
|
||||
"lease_end_date": _iso_date(dates[1]),
|
||||
"lessors": _parties(content, "Лизингодатели"),
|
||||
"lessees": _parties(content, "Лизингополучатели"),
|
||||
"subjects": _subjects(content),
|
||||
"terminated_on": _iso_date(_label_value(content, "Дата прекращения")),
|
||||
"termination_reason": _label_value(content, "Причина прекращения") or None,
|
||||
"related_messages": list(related.values()),
|
||||
"message_text": content.get_text("\n", strip=True),
|
||||
"source_url": url,
|
||||
"fetched_at": timezone.now().isoformat(),
|
||||
}
|
||||
if (
|
||||
not result["lessors"]
|
||||
or not result["lessees"]
|
||||
or any(
|
||||
result[key] is None
|
||||
for key in ("contract_date", "lease_start_date", "lease_end_date")
|
||||
)
|
||||
):
|
||||
raise FedresursUpstreamError("incomplete_leasing_contract")
|
||||
return result
|
||||
|
||||
|
||||
def fetch_leasing_html(driver, url: str) -> str:
|
||||
"""Normal rendered UI navigation; no private endpoints or challenge workaround."""
|
||||
expected_guid = message_guid(url)
|
||||
try:
|
||||
driver.get(url)
|
||||
_challenge(driver.page_source)
|
||||
|
||||
def ready(browser):
|
||||
_challenge(browser.page_source)
|
||||
return browser.find_elements(By.TAG_NAME, "sfact-message-card")
|
||||
|
||||
WebDriverWait(driver, min(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS, 45)).until(
|
||||
ready
|
||||
)
|
||||
if message_guid(driver.current_url) != expected_guid:
|
||||
raise FedresursUpstreamError("leasing_message_redirect_mismatch")
|
||||
html = driver.page_source
|
||||
if len(html.encode("utf-8")) > 4 * 1024 * 1024:
|
||||
raise FedresursUpstreamError("leasing_message_too_large")
|
||||
return html
|
||||
except TimeoutException as exc:
|
||||
raise FedresursUpstreamError("upstream_timeout") from exc
|
||||
except WebDriverException as exc:
|
||||
raise FedresursUpstreamError("browser_failure") from exc
|
||||
|
||||
|
||||
def _due(payload: dict) -> bool:
|
||||
state = payload.get("lease_details_status", {})
|
||||
if state.get("status") == "loaded" and payload.get("lease_details"):
|
||||
return False
|
||||
retry_at = parse_datetime(str(state.get("next_retry_at") or ""))
|
||||
return retry_at is None or retry_at <= timezone.now()
|
||||
|
||||
|
||||
def _save_details(records: list, *, details: dict | None, error: str | None) -> int:
|
||||
updated = 0
|
||||
with transaction.atomic():
|
||||
for original in records:
|
||||
obj = OrganizationSourceRecord.objects.select_for_update().get(
|
||||
pk=original.pk
|
||||
)
|
||||
if obj.payload.get("message_guid") != original.payload.get("message_guid"):
|
||||
continue
|
||||
payload = dict(obj.payload)
|
||||
state = {
|
||||
"status": "failed" if error else "loaded",
|
||||
"checked_at": timezone.now().isoformat(),
|
||||
}
|
||||
if error:
|
||||
state.update(
|
||||
error_code=error,
|
||||
next_retry_at=(timezone.now() + timedelta(hours=24)).isoformat(),
|
||||
)
|
||||
elif details is not None:
|
||||
identifiers = {
|
||||
(p.get("inn"), p.get("ogrn"))
|
||||
for p in details["lessors"] + details["lessees"]
|
||||
}
|
||||
org = obj.extension.organization
|
||||
if (org.inn, org.ogrn) not in identifiers:
|
||||
state.update(
|
||||
status="failed",
|
||||
error_code="leasing_party_mismatch",
|
||||
next_retry_at=(
|
||||
timezone.now() + timedelta(hours=24)
|
||||
).isoformat(),
|
||||
)
|
||||
else:
|
||||
payload["lease_details"] = details
|
||||
updated += 1
|
||||
payload["lease_details_status"] = state
|
||||
obj.payload = payload
|
||||
obj.save(update_fields=("payload", "updated_at"))
|
||||
return updated
|
||||
|
||||
|
||||
def _candidate_groups(record_uid: str | None) -> dict[str, list]:
|
||||
qs = OrganizationSourceRecord.objects.filter(source=SOURCE).select_related(
|
||||
"extension__organization"
|
||||
)
|
||||
target_guid = None
|
||||
if record_uid:
|
||||
try:
|
||||
original = qs.get(uid=record_uid)
|
||||
except OrganizationSourceRecord.DoesNotExist as exc:
|
||||
raise ValueError("leasing_record_not_found") from exc
|
||||
target_guid = message_guid(original.payload.get("source_url") or original.url)
|
||||
groups = {}
|
||||
for obj in qs.order_by("created_at", "uid").iterator(chunk_size=500):
|
||||
try:
|
||||
guid = message_guid(obj.payload.get("source_url") or obj.url)
|
||||
if str(uuid.UUID(obj.payload.get("message_guid", ""))) != guid:
|
||||
continue
|
||||
if target_guid is not None and guid != target_guid:
|
||||
continue
|
||||
except (FedresursUpstreamError, ValueError, TypeError, AttributeError):
|
||||
continue
|
||||
groups.setdefault(guid, []).append(obj)
|
||||
return groups
|
||||
|
||||
|
||||
def _prepare_candidates(record_uid, max_pages, result):
|
||||
candidates = []
|
||||
for records in _candidate_groups(record_uid).values():
|
||||
due = [obj for obj in records if _due(obj.payload)]
|
||||
loaded = next(
|
||||
(
|
||||
obj.payload["lease_details"]
|
||||
for obj in records
|
||||
if obj.payload.get("lease_details_status", {}).get("status") == "loaded"
|
||||
and obj.payload.get("lease_details")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if loaded is not None and due:
|
||||
_apply_details(due, loaded, result)
|
||||
elif due and len(candidates) < max_pages:
|
||||
candidates.append(due)
|
||||
return candidates
|
||||
|
||||
|
||||
def _apply_details(records, details, result):
|
||||
updated = _save_details(records, details=details, error=None)
|
||||
result["records_updated"] += updated
|
||||
if updated != len(records):
|
||||
result["messages_failed"] += 1
|
||||
result["error_code"] = "leasing_party_mismatch"
|
||||
|
||||
|
||||
def _enrich_candidates(candidates, owner, result):
|
||||
# Bound wall time below the lock TTL even if the configured page budget grows.
|
||||
deadline = time.monotonic() + 30 * 60
|
||||
with public_browser(user_agent=settings.FEDRESURS_LEASING_USER_AGENT) as driver:
|
||||
driver.set_page_load_timeout(min(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS, 45))
|
||||
for records in candidates:
|
||||
if cache.get(LOCK_KEY) != owner or time.monotonic() >= deadline:
|
||||
result["error_code"] = "enrichment_lock_or_time_limit"
|
||||
break
|
||||
if result["pages_requested"]:
|
||||
time.sleep(max(2, float(settings.FEDRESURS_MIN_INTERVAL_SECONDS)))
|
||||
url = records[0].payload.get("source_url") or records[0].url
|
||||
result["pages_requested"] += 1
|
||||
try:
|
||||
details = parse_leasing_html(fetch_leasing_html(driver, url), url=url)
|
||||
except FedresursUpstreamError as exc:
|
||||
code = str(exc)
|
||||
_save_details(records, details=None, error=code)
|
||||
result["messages_failed"] += 1
|
||||
result["error_code"] = code
|
||||
if code in SOURCE_ERRORS:
|
||||
cache.set(COOLDOWN_KEY, True, timeout=24 * 3600)
|
||||
break
|
||||
else:
|
||||
_apply_details(records, details, result)
|
||||
|
||||
|
||||
def enrich_leasing_details(
|
||||
*, max_pages: int | None = None, record_uid: str | None = None
|
||||
) -> dict:
|
||||
"""Persist immutable message details once, with a page budget and source cooldown."""
|
||||
limit = min(int(settings.FEDRESURS_LEASING_MAX_PAGES), 100)
|
||||
max_pages = limit if max_pages is None else max_pages
|
||||
if (
|
||||
isinstance(max_pages, bool)
|
||||
or not isinstance(max_pages, int)
|
||||
or not 1 <= max_pages <= limit
|
||||
):
|
||||
raise ValueError("invalid_leasing_page_budget")
|
||||
result = {
|
||||
"pages_requested": 0,
|
||||
"records_updated": 0,
|
||||
"messages_failed": 0,
|
||||
"error_code": None,
|
||||
}
|
||||
if cache.get(COOLDOWN_KEY):
|
||||
return {**result, "error_code": "upstream_cooldown"}
|
||||
owner = str(uuid.uuid4())
|
||||
if not cache.add(LOCK_KEY, owner, timeout=LOCK_SECONDS):
|
||||
return {**result, "error_code": "enrichment_already_running"}
|
||||
try:
|
||||
candidates = _prepare_candidates(record_uid, max_pages, result)
|
||||
if candidates:
|
||||
try:
|
||||
_enrich_candidates(candidates, owner, result)
|
||||
except WebDriverException:
|
||||
result["error_code"] = "browser_failure"
|
||||
cache.set(COOLDOWN_KEY, True, timeout=24 * 3600)
|
||||
return result
|
||||
finally:
|
||||
if result["records_updated"]:
|
||||
invalidate_organization_api_cache()
|
||||
if cache.get(LOCK_KEY) == owner:
|
||||
cache.delete(LOCK_KEY)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""Bounded backfill of existing lease events without any Checko calls."""
|
||||
|
||||
import json
|
||||
|
||||
from apps.parsers.fedresurs_company_profile import FedresursUpstreamError
|
||||
from apps.parsers.leasing_details import enrich_leasing_details
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Дополнить сохранённые события лизинга из публичных сообщений Федресурса"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument("--max-pages", type=int, default=None)
|
||||
parser.add_argument("--record-uid", default=None)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
try:
|
||||
result = enrich_leasing_details(
|
||||
max_pages=options["max_pages"], record_uid=options["record_uid"]
|
||||
)
|
||||
except (ValueError, FedresursUpstreamError) as exc:
|
||||
raise CommandError(str(exc)) from exc
|
||||
self.stdout.write(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
||||
if result["error_code"]:
|
||||
raise CommandError(result["error_code"])
|
||||
@@ -529,6 +529,13 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=30),
|
||||
refresh_params=(
|
||||
RefreshParamDefinition(
|
||||
name="mode",
|
||||
label="Режим обновления",
|
||||
description="collection: список Checko; details: дополнить сохранённые события из Федресурса без запросов Checko.",
|
||||
),
|
||||
),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="checko-messages",
|
||||
@@ -878,6 +885,13 @@ class SourceCardService:
|
||||
)
|
||||
else:
|
||||
params = cls._validate_refresh_params(definition, params)
|
||||
if slug == "checko-leasing" and params.get("mode", "collection") not in (
|
||||
"collection",
|
||||
"details",
|
||||
):
|
||||
raise APIValidationError(
|
||||
message="Допустимые режимы: collection, details.", code="invalid_mode"
|
||||
)
|
||||
if mode is not None:
|
||||
allowed_modes = (
|
||||
("incremental", "full")
|
||||
@@ -1449,6 +1463,8 @@ class SourceCardService:
|
||||
),
|
||||
)
|
||||
elif definition.slug == "checko-leasing":
|
||||
if "mode" in params:
|
||||
kwargs["mode"] = params["mode"]
|
||||
task_specs = (
|
||||
(
|
||||
parse_checko_leasing,
|
||||
|
||||
@@ -379,7 +379,13 @@ def parse_fssp_enforcement_proceedings(
|
||||
soft_time_limit=8 * 3600,
|
||||
time_limit=8 * 3600 + 300,
|
||||
)
|
||||
def parse_checko_leasing(self, *, requested_by_id: int | None = None) -> dict:
|
||||
def parse_checko_leasing(
|
||||
self, *, requested_by_id: int | None = None, mode: str = "collection"
|
||||
) -> dict:
|
||||
if mode == "details":
|
||||
return _run_leasing_details(self, requested_by_id=requested_by_id)
|
||||
if mode != "collection":
|
||||
raise SnapshotValidationError("invalid_mode")
|
||||
from apps.parsers.checko_section_snapshots import refresh_checko_section
|
||||
|
||||
return _run_snapshot(
|
||||
@@ -391,6 +397,43 @@ def parse_checko_leasing(self, *, requested_by_id: int | None = None) -> dict:
|
||||
)
|
||||
|
||||
|
||||
def _run_leasing_details(task, *, requested_by_id: int | None) -> dict:
|
||||
"""Detail-only job; never create a collection log or advance full snapshot success."""
|
||||
from apps.parsers.leasing_details import enrich_leasing_details
|
||||
|
||||
job, _ = BackgroundJob.objects.get_or_create(
|
||||
task_id=task.request.id or str(uuid.uuid4()),
|
||||
defaults={
|
||||
"task_name": task.name,
|
||||
"user_id": requested_by_id,
|
||||
"meta": {"source": "checko_leasing", "mode": "details"},
|
||||
},
|
||||
)
|
||||
try:
|
||||
if not _claim_snapshot_job(job, "checko_leasing", task.name):
|
||||
job.refresh_from_db()
|
||||
return job.result or {"status": job.status, "duplicate_delivery": True}
|
||||
job.update_progress(5, "Дополнение сохранённых событий лизинга")
|
||||
result = enrich_leasing_details()
|
||||
job.refresh_from_db()
|
||||
if job.status != JobStatus.STARTED:
|
||||
raise SnapshotValidationError("snapshot_job_no_longer_active")
|
||||
job.meta = {**job.meta, "mode": "details", **result}
|
||||
job.result = {"source": "checko_leasing", "mode": "details", **result}
|
||||
job.save(update_fields=("meta", "result", "updated_at"))
|
||||
if result["error_code"] and not result["records_updated"]:
|
||||
raise SnapshotValidationError(result["error_code"])
|
||||
status = "partial" if result["error_code"] else "success"
|
||||
job.update_progress(100, "Порция дополнения лизинга завершена")
|
||||
job.complete(result={"status": status, **job.result})
|
||||
if result["records_updated"]:
|
||||
transaction.on_commit(_refresh_published_exports)
|
||||
return job.result
|
||||
except Exception as exc:
|
||||
job.fail(error=_snapshot_error_message(exc))
|
||||
raise
|
||||
|
||||
|
||||
@shared_task(
|
||||
bind=True,
|
||||
name="parsers.checko_messages.refresh",
|
||||
|
||||
@@ -22282,6 +22282,12 @@
|
||||
},
|
||||
"lease_event_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"lease_details": {
|
||||
"$ref": "#/components/schemas/LeaseDetails"
|
||||
},
|
||||
"lease_details_status": {
|
||||
"$ref": "#/components/schemas/LeaseDetailsStatus"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
@@ -22512,6 +22518,184 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"LeaseParty": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"inn": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"ogrn": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"inn",
|
||||
"ogrn"
|
||||
]
|
||||
},
|
||||
"LeaseSubject": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"identifier": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"identifier_raw": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"classifier_code": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"classifier_name": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"identifier",
|
||||
"identifier_raw",
|
||||
"classifier_code",
|
||||
"classifier_name",
|
||||
"description"
|
||||
]
|
||||
},
|
||||
"LeaseRelatedMessage": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"guid": {
|
||||
"type": "string"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
},
|
||||
"label": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"guid",
|
||||
"url",
|
||||
"label"
|
||||
]
|
||||
},
|
||||
"LeaseDetails": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"contract_number": {
|
||||
"type": "string"
|
||||
},
|
||||
"contract_date": {
|
||||
"type": "string",
|
||||
"format": "date"
|
||||
},
|
||||
"lease_start_date": {
|
||||
"type": "string",
|
||||
"format": "date"
|
||||
},
|
||||
"lease_end_date": {
|
||||
"type": "string",
|
||||
"format": "date"
|
||||
},
|
||||
"lessors": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/LeaseParty"
|
||||
}
|
||||
},
|
||||
"lessees": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/LeaseParty"
|
||||
}
|
||||
},
|
||||
"subjects": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/LeaseSubject"
|
||||
}
|
||||
},
|
||||
"terminated_on": {
|
||||
"type": "string",
|
||||
"nullable": true,
|
||||
"format": "date"
|
||||
},
|
||||
"termination_reason": {
|
||||
"type": "string",
|
||||
"nullable": true
|
||||
},
|
||||
"related_messages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/LeaseRelatedMessage"
|
||||
}
|
||||
},
|
||||
"message_text": {
|
||||
"type": "string"
|
||||
},
|
||||
"source_url": {
|
||||
"type": "string"
|
||||
},
|
||||
"fetched_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"contract_number",
|
||||
"contract_date",
|
||||
"lease_start_date",
|
||||
"lease_end_date",
|
||||
"lessors",
|
||||
"lessees",
|
||||
"subjects",
|
||||
"terminated_on",
|
||||
"termination_reason",
|
||||
"related_messages",
|
||||
"message_text",
|
||||
"source_url",
|
||||
"fetched_at"
|
||||
]
|
||||
},
|
||||
"LeaseDetailsStatus": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"status": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"loaded",
|
||||
"failed"
|
||||
]
|
||||
},
|
||||
"checked_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
},
|
||||
"error_code": {
|
||||
"type": "string"
|
||||
},
|
||||
"next_retry_at": {
|
||||
"type": "string",
|
||||
"format": "date-time"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"status",
|
||||
"checked_at"
|
||||
]
|
||||
},
|
||||
"ExchangeInstrumentRecordList": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
||||
@@ -18,8 +18,53 @@ class CheckoMessagePayloadSerializer(serializers.Serializer):
|
||||
upstream = serializers.JSONField()
|
||||
|
||||
|
||||
class LeasePartySerializer(serializers.Serializer):
|
||||
name = serializers.CharField(allow_null=True)
|
||||
inn = serializers.CharField(allow_null=True)
|
||||
ogrn = serializers.CharField(allow_null=True)
|
||||
|
||||
|
||||
class LeaseSubjectSerializer(serializers.Serializer):
|
||||
identifier = serializers.CharField(allow_null=True)
|
||||
identifier_raw = serializers.CharField(allow_null=True)
|
||||
classifier_code = serializers.CharField(allow_null=True)
|
||||
classifier_name = serializers.CharField(allow_null=True)
|
||||
description = serializers.CharField(allow_null=True)
|
||||
|
||||
|
||||
class LeaseRelatedMessageSerializer(serializers.Serializer):
|
||||
guid = serializers.CharField()
|
||||
url = serializers.CharField()
|
||||
label = serializers.CharField()
|
||||
|
||||
|
||||
class LeaseDetailsSerializer(serializers.Serializer):
|
||||
contract_number = serializers.CharField()
|
||||
contract_date = serializers.DateField()
|
||||
lease_start_date = serializers.DateField()
|
||||
lease_end_date = serializers.DateField()
|
||||
lessors = LeasePartySerializer(many=True)
|
||||
lessees = LeasePartySerializer(many=True)
|
||||
subjects = LeaseSubjectSerializer(many=True)
|
||||
terminated_on = serializers.DateField(allow_null=True)
|
||||
termination_reason = serializers.CharField(allow_null=True)
|
||||
related_messages = LeaseRelatedMessageSerializer(many=True)
|
||||
message_text = serializers.CharField()
|
||||
source_url = serializers.CharField()
|
||||
fetched_at = RegistryUTCDateTimeField()
|
||||
|
||||
|
||||
class LeaseDetailsStatusSerializer(serializers.Serializer):
|
||||
status = serializers.ChoiceField(choices=("loaded", "failed"))
|
||||
checked_at = RegistryUTCDateTimeField()
|
||||
error_code = serializers.CharField(required=False)
|
||||
next_retry_at = RegistryUTCDateTimeField(required=False)
|
||||
|
||||
|
||||
class CheckoLeasingPayloadSerializer(CheckoMessagePayloadSerializer):
|
||||
lease_event_type = serializers.CharField()
|
||||
lease_details = LeaseDetailsSerializer(required=False)
|
||||
lease_details_status = LeaseDetailsStatusSerializer(required=False)
|
||||
|
||||
|
||||
class CheckoLicensePayloadSerializer(serializers.Serializer):
|
||||
|
||||
@@ -346,6 +346,12 @@ FEDRESURS_BROWSER_DRIVER = os.getenv(
|
||||
FEDRESURS_PAGE_TIMEOUT_SECONDS = int(os.getenv("FEDRESURS_PAGE_TIMEOUT_SECONDS", "45"))
|
||||
FEDRESURS_MIN_INTERVAL_SECONDS = float(os.getenv("FEDRESURS_MIN_INTERVAL_SECONDS", "2"))
|
||||
FEDRESURS_MAX_RETRIES = int(os.getenv("FEDRESURS_MAX_RETRIES", "2"))
|
||||
FEDRESURS_LEASING_MAX_PAGES = int(os.getenv("FEDRESURS_LEASING_MAX_PAGES", "25"))
|
||||
FEDRESURS_LEASING_USER_AGENT = os.getenv(
|
||||
"FEDRESURS_LEASING_USER_AGENT",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/154.0.0.0 Safari/537.36 ECOS-LeasingCollector/1.0",
|
||||
)
|
||||
|
||||
# Default primary key field type
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
|
||||
|
||||
Reference in New Issue
Block a user