diff --git a/.dockerignore b/.dockerignore index f5df1e3..209bbee 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,13 @@ .qoder .zed .venv +.env +.env.* +**/.env +**/.env.* +**/secrets +**/*.pem +**/*.key __pycache__ *.pyc *.pyo diff --git a/.env.prod.example b/.env.prod.example index 3ce7abb..8ef40de 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -35,6 +35,14 @@ DATA_UPLOAD_MAX_MEMORY_SIZE=220200960 # Example: PARSER_PROXIES=http://user:pass@proxy1:8080,http://user:pass@proxy2:8080 PARSER_PROXIES= +# Public-UI Fedresurs collection; enable only after a successful full dev run. +FEDRESURS_COMPANY_PROFILE_ENABLED=false +FEDRESURS_BROWSER_BINARY=/opt/chrome/chrome-linux64/chrome +FEDRESURS_BROWSER_DRIVER=/opt/chrome/chromedriver-linux64/chromedriver +FEDRESURS_PAGE_TIMEOUT_SECONDS=45 +FEDRESURS_MIN_INTERVAL_SECONDS=2 +FEDRESURS_MAX_RETRIES=2 + # 1 to collect static files during migrate service, 0 to skip COLLECTSTATIC_ON_MIGRATE=0 diff --git a/docker/Dockerfile b/docker/Dockerfile index 8dd1b95..4463f89 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -139,12 +139,25 @@ RUN apt-get update \ libpango-1.0-0 \ libcairo2 \ libatspi2.0-0 \ + unzip \ && rm -rf /var/lib/apt/lists/* ENV PLAYWRIGHT_BROWSERS_PATH=/app/.playwright RUN python -m playwright install chromium \ && chown -R appuser:appgroup /app/.playwright +# Chrome for Testing publishes matching browser and driver archives for one +# immutable version. Selenium uses only this pair, not a host browser/profile. +ARG FEDRESURS_CHROME_VERSION=154.0.8037.57 +RUN mkdir -p /opt/chrome \ + && curl -fSL "https://storage.googleapis.com/chrome-for-testing-public/${FEDRESURS_CHROME_VERSION}/linux64/chrome-linux64.zip" -o /tmp/chrome.zip \ + && curl -fSL "https://storage.googleapis.com/chrome-for-testing-public/${FEDRESURS_CHROME_VERSION}/linux64/chromedriver-linux64.zip" -o /tmp/chromedriver.zip \ + && unzip -q /tmp/chrome.zip -d /opt/chrome \ + && unzip -q /tmp/chromedriver.zip -d /opt/chrome \ + && chmod +x /opt/chrome/chrome-linux64/chrome /opt/chrome/chromedriver-linux64/chromedriver \ + && rm /tmp/chrome.zip /tmp/chromedriver.zip \ + && chown -R appuser:appgroup /opt/chrome + USER appuser diff --git a/docs/fedresurs-company-profile.md b/docs/fedresurs-company-profile.md new file mode 100644 index 0000000..e342dbe --- /dev/null +++ b/docs/fedresurs-company-profile.md @@ -0,0 +1,35 @@ +# Публичная карточка Федресурса + +Источник `fedresurs-company-profile` (`fedresurs_company_profile`) открывает +новую анонимную сессию Selenium с закреплённой парой Chrome for Testing и +chromedriver. Поиск идёт через публичную форму по ИНН, подтверждается точным +совпадением ИНН и ОГРН, затем браузер читает `/companies/{guid}`. Код не +обращается к `/backend`, не переносит cookies пользователей и не обходит +CAPTCHA/Qrator. + +Для каждой карточки сохраняется вся отрендеренная страница и каждый +обнаруженный раздел, включая новые разделы с неизвестным заранее именем: +полный текст, +иерархия DOM без нестабильных Angular/CSS-атрибутов, пары `label/value` и +публичные ссылки. Адаптер проходит обнаруженные ссылки полного списка +публикаций и правопередач, проверяет заявленное число записей и открывает +каждое обнаруженное публичное сообщение. Полный текст, поля и ссылки этих +страниц входят в detail API. Бинарные документы не скачиваются; их метаданные +и ссылки сохраняются. Неизвестный подраздел, неполная выдача, challenge, +таймаут или превышение лимита страницы являются технической ошибкой: batch не +публикуется и предыдущий snapshot остаётся доступен. + +В XLSX длинные значения (в том числе полный текст страницы) не обрезаются: +основная ячейка ссылается на листы `overflow_values_N`. Их строки с теми же +`record_uid` и `field_name` нужно упорядочить по `part`, убрать префикс `b64:` +у каждого `chunk`, склеить части и декодировать как base64/UTF-8. JSON-выгрузка +содержит значение непосредственно. + +`POST /api/v1/parsers/run/fedresurs_company_profile/` запускает `full` по всему +активному каноническому реестру или `targeted` с массивом до 500 +`organization_uids`. Организации обрабатываются последовательно. Параметры +`FEDRESURS_*` задают feature flag, браузер, timeout, паузу и retries. +Ежедневная задача `02:30 Europe/Moscow` устанавливается миграцией выключенной; +её можно включить только после полного успешного ручного запуска на dev. +No-match и неоднозначные результаты фиксируются в quarantine, а не трактуются +как ошибки транспорта. Upload для этого источника возвращает HTTP 405. diff --git a/src/apps/parsers/api_result_urls.py b/src/apps/parsers/api_result_urls.py index 4257860..be00aa9 100644 --- a/src/apps/parsers/api_result_urls.py +++ b/src/apps/parsers/api_result_urls.py @@ -1,11 +1,14 @@ """Frontend-oriented parser result API v1 routes.""" from apps.parsers.serializers import ( + IntegratedStructureUploadResponseSerializer, ParserResultRecordSerializer, ParserRunResponseSerializer, ) from apps.parsers.source_registry import PARSER_SOURCES, ParserSourceDescriptor from apps.parsers.views import ( + INTEGRATED_ATTACHMENT_PARAM, + INTEGRATED_WORKBOOK_PARAM, MEDIA_NEWS_UPLOAD_FILE_PARAM, RESULT_DETAIL_PARAMS, RESULT_LIST_PARAMS, @@ -125,23 +128,30 @@ def _upload_view(descriptor: ParserSourceDescriptor): f"Ручная загрузка файла для источника: {descriptor.data_scope}. " "Файл обрабатывается через Celery." ), - manual_parameters=[ - ( + manual_parameters=( + [INTEGRATED_WORKBOOK_PARAM, INTEGRATED_ATTACHMENT_PARAM] + if descriptor.key == "integrated_structure_documents" + else [ MEDIA_NEWS_UPLOAD_FILE_PARAM if descriptor.key == "media_news" else ROPK_SANCTIONS_UPLOAD_FILE_PARAM if descriptor.key == "ropk_sanctions" else UPLOAD_FILE_PARAM - ) - ], + ] + ), consumes=["multipart/form-data"], tags=[tag], responses={ - 202: ParserRunResponseSerializer, + (200 if descriptor.key == "integrated_structure_documents" else 202): ( + IntegratedStructureUploadResponseSerializer + if descriptor.key == "integrated_structure_documents" + else ParserRunResponseSerializer + ), 400: "Ошибка валидации", 409: "Загрузка уже выполняется", 413: "Файл превышает лимит", 415: "Неподдерживаемый тип файла", + 422: "Неверные листы, идентификаторы или вложения", }, ) def post(self, request): diff --git a/src/apps/parsers/artifact_storage.py b/src/apps/parsers/artifact_storage.py index 980fe22..ee8221e 100644 --- a/src/apps/parsers/artifact_storage.py +++ b/src/apps/parsers/artifact_storage.py @@ -7,7 +7,11 @@ from django.core.exceptions import ImproperlyConfigured from django.core.files.storage import FileSystemStorage, Storage, default_storage from django.utils.deconstruct import deconstructible -SRO_ARTIFACT_PREFIX = "parser-artifacts/sro_membership_check/" +PRIVATE_ARTIFACT_PREFIXES = ( + "parser-artifacts/sro_membership_check/", + "parser-artifacts/integrated_structure_documents/", + "parser-artifacts/fedresurs_company_profile/", +) @deconstructible @@ -16,7 +20,7 @@ class ParserArtifactStorage(Storage): @staticmethod def _storage(name): - if not str(name).startswith(SRO_ARTIFACT_PREFIX): + if not str(name).startswith(PRIVATE_ARTIFACT_PREFIXES): return default_storage configured = Path(settings.PARSER_PRIVATE_ARTIFACT_ROOT) if not configured.is_absolute(): @@ -60,7 +64,7 @@ class ParserArtifactStorage(Storage): return self._storage(name).path(name) def url(self, name): - if str(name).startswith(SRO_ARTIFACT_PREFIX): + if str(name).startswith(PRIVATE_ARTIFACT_PREFIXES): raise ValueError("Private parser artifacts have no public URL") return self._storage(name).url(name) diff --git a/src/apps/parsers/fedresurs_browser.py b/src/apps/parsers/fedresurs_browser.py new file mode 100644 index 0000000..5678d16 --- /dev/null +++ b/src/apps/parsers/fedresurs_browser.py @@ -0,0 +1,261 @@ +"""Sequential anonymous Selenium navigation of the public Fedresurs UI.""" + +from __future__ import annotations + +import re +import time +from collections.abc import Iterator +from contextlib import contextmanager, suppress +from urllib.parse import urlparse + +from apps.parsers.fedresurs_company_profile import ( + FEDRESURS_ORIGIN, + FedresursUpstreamError, + _challenge, + _label_value, + parse_profile_html, + parse_related_page_html, + parse_search_html, +) +from bs4 import BeautifulSoup +from django.conf import settings +from selenium import webdriver +from selenium.common.exceptions import TimeoutException, WebDriverException +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait + + +def _wait_public_element(browser, by: str, value: str): + """Keep checking the page for a late challenge while Angular renders.""" + _challenge(browser.page_source) + elements = browser.find_elements(by, value) + return elements[0] if elements else False + + +@contextmanager +def public_browser() -> Iterator[webdriver.Chrome]: + """Start a fresh browser, with no inherited profile, cookies, or stealth flags.""" + options = webdriver.ChromeOptions() + options.binary_location = settings.FEDRESURS_BROWSER_BINARY + options.add_argument("--headless=new") + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + driver = webdriver.Chrome( + service=Service(executable_path=settings.FEDRESURS_BROWSER_DRIVER), + options=options, + ) + driver.set_page_load_timeout(settings.FEDRESURS_PAGE_TIMEOUT_SECONDS) + try: + yield driver + finally: + driver.quit() + + +def _exact_card_index(cards, *, inn: str, ogrn: str) -> int: + matches = [] + for index, card in enumerate(cards): + soup = BeautifulSoup(card.get_attribute("outerHTML"), "html.parser") + if _label_value(soup, "ИНН") == inn and _label_value(soup, "ОГРН") == ogrn: + matches.append(index) + if len(matches) != 1: + raise FedresursUpstreamError("search_result_changed") + return matches[0] + + +def _related_company_urls(profile: dict) -> list[tuple[str, str]]: + guid = profile["fedresurs_guid"] + prefix = f"/companies/{guid}/" + urls = set() + for section in profile["rendered_sections"].values(): + for link in section["links"]: + parsed = urlparse(link["url"]) + if parsed.scheme != "https" or parsed.netloc != "fedresurs.ru": + continue + if parsed.path.startswith(prefix): + page_type = parsed.path[len(prefix) :].strip("/") + if page_type: + urls.add((page_type, f"{FEDRESURS_ORIGIN}{parsed.path}")) + return sorted(urls) + + +def _collect_listing(driver, wait, *, page_type: str, url: str) -> dict: + driver.get(url) + _challenge(driver.page_source) + root_tag = ( + "app-company-card-publications" + if page_type == "publications" + else "app-company-card-rights-transfers" + if page_type == "rightstransfers" + else None + ) + if root_tag is None: + raise FedresursUpstreamError("unsupported_public_section") + wait.until(lambda browser: _wait_public_element(browser, By.TAG_NAME, root_tag)) + wait.until( + lambda browser: _wait_public_element(browser, By.TAG_NAME, "app-found-records") + ) + _challenge(driver.page_source) + found = driver.find_element(By.TAG_NAME, "app-found-records").text + count_match = re.search(r"(\d[\d\s]*)", found) + if count_match is None: + raise FedresursUpstreamError("incomplete_related_page") + reported = int(re.sub(r"\s", "", count_match.group(1))) + card_tag = ( + "entity-card-publications-search-result-card" + if page_type == "publications" + else "company-rights-transfers-result-card" + ) + snapshots = [] + seen_cards = set() + for _ in range(max(2, reported + 2)): + for card in driver.find_elements(By.TAG_NAME, card_tag): + card_html = card.get_attribute("outerHTML") + if card_html not in seen_cards: + seen_cards.add(card_html) + snapshots.append( + parse_related_page_html( + f"<{root_tag}>{card_html}", + url=driver.current_url, + page_type=page_type, + ) + ) + if len(snapshots) >= reported: + break + before = len(snapshots) + driver.execute_script("window.scrollTo(0, document.body.scrollHeight)") + time.sleep(max(0.2, float(settings.FEDRESURS_MIN_INTERVAL_SECONDS))) + try: + wait.until( + lambda browser, seen=before: len( + browser.find_elements(By.TAG_NAME, card_tag) + ) > seen + ) + except TimeoutException as exc: + raise FedresursUpstreamError("incomplete_related_page") from exc + if len(snapshots) != reported: + raise FedresursUpstreamError("incomplete_related_page") + page = parse_related_page_html( + driver.page_source, url=driver.current_url, page_type=page_type + ) + page["reported_records_count"] = reported + page["records"] = snapshots + return page + + +def _collect_related_pages(driver, wait, profile: dict) -> None: + pages = {} + message_urls = set() + for page_type, url in _related_company_urls(profile): + time.sleep(max(0, float(settings.FEDRESURS_MIN_INTERVAL_SECONDS))) + page = _collect_listing(driver, wait, page_type=page_type, url=url) + pages[page_type] = page + for record in page["records"]: + for link in record["links"]: + parsed = urlparse(link["url"]) + if parsed.netloc == "fedresurs.ru" and re.fullmatch( + r"/sfactmessages/[0-9a-f-]{36}", parsed.path, re.IGNORECASE + ): + message_urls.add(f"{FEDRESURS_ORIGIN}{parsed.path}") + messages = [] + for url in sorted(message_urls): + time.sleep(max(0, float(settings.FEDRESURS_MIN_INTERVAL_SECONDS))) + driver.get(url) + _challenge(driver.page_source) + wait.until( + lambda browser: _wait_public_element(browser, By.TAG_NAME, "sfact-message-card") + ) + messages.append( + parse_related_page_html( + driver.page_source, url=driver.current_url, page_type="message" + ) + ) + profile["related_pages"] = pages + profile["message_details"] = messages + + +def lookup_company( + driver: webdriver.Chrome, *, inn: str, ogrn: str +) -> tuple[str, dict | None]: + """Return (coverage, profile), never interpreting a challenge as no-match.""" + timeout = settings.FEDRESURS_PAGE_TIMEOUT_SECONDS + base_handle = driver.current_window_handle + try: + driver.get(FEDRESURS_ORIGIN + "/") + _challenge(driver.page_source) + wait = WebDriverWait(driver, timeout) + field = wait.until( + lambda browser: _wait_public_element( + browser, By.CSS_SELECTOR, 'input[formcontrolname="searchString"]' + ) + ) + field.clear() + field.send_keys(inn) + form = field.find_element(By.XPATH, "./ancestor::form") + form.find_element(By.TAG_NAME, "button").click() + wait.until( + lambda browser: _wait_public_element( + browser, By.TAG_NAME, "app-quick-search-result" + ) + ) + # The result component is mounted before the asynchronous search ends. + # Require a stable rendered result before classifying an empty set. + last_html = "" + stable_since = time.monotonic() + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result_nodes = driver.find_elements(By.TAG_NAME, "app-quick-search-result") + html = result_nodes[0].get_attribute("outerHTML") if result_nodes else "" + if html == last_html: + if time.monotonic() - stable_since >= 1.0: + break + else: + last_html = html + stable_since = time.monotonic() + time.sleep(0.2) + else: + raise FedresursUpstreamError("search_timeout") + _challenge(driver.page_source) + classification = parse_search_html(last_html, inn=inn, ogrn=ogrn) + if classification != "match": + return classification, None + cards = driver.find_elements( + By.TAG_NAME, "app-quick-search-entities-result-card" + ) + index = _exact_card_index(cards, inn=inn, ogrn=ogrn) + old_handles = set(driver.window_handles) + cards[index].click() + new_handle = wait.until( + lambda browser: next( + ( + handle + for handle in browser.window_handles + if handle not in old_handles + ), + None, + ) + ) + driver.switch_to.window(new_handle) + wait.until( + lambda browser: _wait_public_element( + browser, By.CSS_SELECTOR, '[page-item-name="general-info"]' + ) + ) + profile = parse_profile_html( + driver.page_source, url=driver.current_url, inn=inn, ogrn=ogrn + ) + _collect_related_pages(driver, wait, profile) + return "match", profile + except FedresursUpstreamError: + raise + except TimeoutException as exc: + raise FedresursUpstreamError("upstream_timeout") from exc + except WebDriverException as exc: + raise FedresursUpstreamError("browser_failure") from exc + finally: + with suppress(WebDriverException): + for handle in driver.window_handles: + if handle != base_handle: + driver.switch_to.window(handle) + driver.close() + driver.switch_to.window(base_handle) diff --git a/src/apps/parsers/fedresurs_company_profile.py b/src/apps/parsers/fedresurs_company_profile.py new file mode 100644 index 0000000..939c83a --- /dev/null +++ b/src/apps/parsers/fedresurs_company_profile.py @@ -0,0 +1,269 @@ +"""Public rendered-UI adapter for Fedresurs company profiles. + +No private backend endpoints, browser cookies, challenge workarounds, or +fingerprint changes are used here. The browser is a fresh anonymous session. +""" + +from __future__ import annotations + +import hashlib +import re +from datetime import datetime +from urllib.parse import urljoin, urlparse + +from bs4 import BeautifulSoup + +FEDRESURS_ORIGIN = "https://fedresurs.ru" +COMPANY_PATH = re.compile( + r"^/companies/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$", + re.IGNORECASE, +) +SECTION_NAMES = { + "general-info": "profile", + "messages": "publications", + "encumbrances": "encumbrances", + "eio": "executive_bodies", + "sro": "sro", + "sro-membership": "sro_memberships", + "licenses": "licenses", + "networth": "net_worth", + "reorganization": "reorganizations", + "bankrupt": "bankruptcy", + "trades": "biddings", +} +CHALLENGE_WORDS = ("captcha", "qrator", "access denied", "проверка браузера") +ACCESS_BLOCK_WORDS = ("403 error", "access to fedresurs.ru is forbidden") +MAX_RENDERED_SECTION_BYTES = 4 * 1024 * 1024 +PUBLIC_PAGE_ROOTS = { + "publications": "app-company-card-publications", + "rightstransfers": "app-company-card-rights-transfers", +} + + +class FedresursUpstreamError(RuntimeError): + """Challenge, timeout, or incomplete public page; never publish as no-match.""" + + +def _challenge(html: str) -> None: + # Script bundles may mention CAPTCHA/Qrator without a challenge being shown. + # Inspect only text visible on the rendered page; missing app roots are + # separately classified as incomplete upstream responses. + visible_text = BeautifulSoup(html, "html.parser") + for hidden in visible_text.find_all(("script", "style", "template", "noscript")): + hidden.decompose() + text = visible_text.get_text(" ", strip=True).casefold() + if any(word in text for word in ACCESS_BLOCK_WORDS): + raise FedresursUpstreamError("upstream_access_blocked") + if any(word in text for word in CHALLENGE_WORDS): + raise FedresursUpstreamError("upstream_challenge") + + +def _label_value(container, label: str) -> str: + """Read the visible value next to a semantic label, not a layout class.""" + for node in container.find_all(string=True): + if str(node).strip().casefold() != label.casefold(): + continue + parent = node.parent + if parent is None or parent.parent is None: + continue + siblings = parent.parent.find_all(recursive=False) + try: + index = siblings.index(parent) + except ValueError: + continue + if index + 1 < len(siblings): + return siblings[index + 1].get_text(" ", strip=True) + return "" + + +def parse_search_html(html: str, *, inn: str, ogrn: str) -> str: + """Classify exact public search cards; no fuzzy company matching.""" + _challenge(html) + soup = BeautifulSoup(html, "html.parser") + cards = soup.find_all("app-quick-search-entities-result-card") + exact = [ + card + for card in cards + if _label_value(card, "ИНН") == inn and _label_value(card, "ОГРН") == ogrn + ] + if len(exact) > 1: + return "ambiguous" + return "match" if exact else "no_match" + + +def _iso_date(value: str) -> str | None: + try: + return datetime.strptime(value, "%d.%m.%Y").date().isoformat() + except ValueError: + return None + + +def _semantic_tree(node, *, base_url: str): + """Retain every rendered text leaf and business-relevant HTML attribute.""" + from bs4.element import NavigableString, Tag + + if isinstance(node, NavigableString): + value = " ".join(str(node).split()) + return value or None + if not isinstance(node, Tag) or node.name in ( + "script", + "style", + "noscript", + "svg", + "template", + ): + return None + attributes = {} + for key, value in node.attrs.items(): + if key.startswith("_ng") or key in ("class", "style") or key.startswith("on"): + continue + if key in ("href", "src"): + value = urljoin(base_url, str(value)) + if urlparse(value).scheme not in ("https", "http"): + continue + attributes[key] = value + children = [ + item + for child in node.children + if (item := _semantic_tree(child, base_url=base_url)) is not None + ] + if not children and not attributes: + return None + return {"tag": node.name, "attributes": attributes, "children": children} + + +def rendered_snapshot(node, *, base_url: str) -> dict: + """Lossless rendered text/tree plus canonical links, without layout selectors.""" + import json + + tree = _semantic_tree(node, base_url=base_url) + if tree is None: + raise FedresursUpstreamError("empty_rendered_section") + encoded = json.dumps(tree, ensure_ascii=False, separators=(",", ":")) + if len(encoded.encode("utf-8")) > MAX_RENDERED_SECTION_BYTES: + raise FedresursUpstreamError("rendered_section_too_large") + links = [] + for link in node.find_all("a", href=True): + href = urljoin(base_url, link["href"]) + if urlparse(href).scheme in ("https", "http"): + links.append({"text": link.get_text(" ", strip=True), "url": href}) + attributes = [] + for candidate in node.find_all(("div", "section", "tr", "dl")): + children = [ + child for child in candidate.children if getattr(child, "name", None) + ] + if len(children) != 2: + continue + label = children[0].get_text(" ", strip=True) + value = children[1].get_text(" ", strip=True) + if not label or not value or len(label) > 100 or len(value) > 2000: + continue + if label == value: + continue + attributes.append({"label": label, "value": value}) + return { + "text": node.get_text("\n", strip=True), + "tree": tree, + "attributes": attributes, + "links": list({item["url"]: item for item in links}.values()), + } + + +def parse_related_page_html(html: str, *, url: str, page_type: str) -> dict: + """Capture complete public company list or message detail as rendered DOM.""" + _challenge(html) + parsed_url = urlparse(url) + if parsed_url.scheme != "https" or parsed_url.netloc != "fedresurs.ru": + raise FedresursUpstreamError("invalid_related_page_url") + soup = BeautifulSoup(html, "html.parser") + root_name = PUBLIC_PAGE_ROOTS.get( + page_type, "sfact-message-card" if page_type == "message" else "" + ) + root = soup.find(root_name) if root_name else None + if root is None: + raise FedresursUpstreamError("incomplete_related_page") + snapshot = rendered_snapshot(root, base_url=url) + snapshot["url"] = url + snapshot["page_type"] = page_type + return snapshot + + +def parse_profile_html(html: str, *, url: str, inn: str, ogrn: str) -> dict: + """Validate a rendered company page and preserve section coverage.""" + _challenge(html) + parsed_url = urlparse(url) + match = COMPANY_PATH.fullmatch(parsed_url.path) + if ( + parsed_url.scheme != "https" + or parsed_url.netloc != "fedresurs.ru" + or match is None + ): + raise FedresursUpstreamError("invalid_company_url") + soup = BeautifulSoup(html, "html.parser") + sections = { + section.get("page-item-name"): section + for section in soup.select("[page-item-name]") + } + if any(name not in sections for name in SECTION_NAMES): + raise FedresursUpstreamError("incomplete_company_page") + general = sections["general-info"] + if _label_value(general, "ИНН") != inn or _label_value(general, "ОГРН") != ogrn: + raise FedresursUpstreamError("company_identity_mismatch") + full_name = _label_value(general, "Полное наименование") + if not full_name: + raise FedresursUpstreamError("incomplete_company_profile") + section_states = { + SECTION_NAMES.get(name, name): "empty" + if "Нет данных" in section.get_text(" ", strip=True) + else "loaded" + for name, section in sections.items() + } + for optional_name in ( + "pledged_subjects", + "account_receivables", + "russian_post_estates", + ): + section_states.setdefault(optional_name, "not_available") + body_text = soup.get_text(" ", strip=True) + if "В процессе ликвидации" in body_text: + status, status_code = "В процессе ликвидации", "liquidating" + elif "Действующее" in body_text: + status, status_code = "Действующее", "active" + elif "Прекращено" in body_text: + status, status_code = "Прекращено", "terminated" + else: + status, status_code = "Не определён", None + return { + "fedresurs_guid": match[1].lower(), + "source_url": url, + "sections": section_states, + "rendered_page": rendered_snapshot(soup.body or soup, base_url=url), + "rendered_sections": { + SECTION_NAMES.get(name, name): rendered_snapshot(section, base_url=url) + for name, section in sections.items() + }, + "content_hash": hashlib.sha256(html.encode("utf-8")).hexdigest(), + "profile": { + "full_name": full_name, + "inn": inn, + "ogrn": ogrn, + "kpp": _label_value(general, "КПП") or None, + "egrul_address": _label_value(general, "Адрес по данным ЕГРЮЛ") or None, + "company_address": _label_value(general, "Адрес по данным компании") + or None, + "authorized_capital": _label_value(general, "Уставный капитал") or None, + "registration_date": _iso_date(_label_value(general, "Дата регистрации")), + "status": { + "name": status, + "is_active": status_code == "active", + "code": status_code, + }, + }, + "publication_count": len( + sections["messages"].select('a[href^="/sfactmessages/"]') + ), + "reorganization_count": sections["reorganization"] + .get_text(" ", strip=True) + .count("Дата события"), + "executive_body_name": _label_value(sections["eio"], "Наименование") or None, + } diff --git a/src/apps/parsers/fedresurs_snapshot.py b/src/apps/parsers/fedresurs_snapshot.py new file mode 100644 index 0000000..35c63a2 --- /dev/null +++ b/src/apps/parsers/fedresurs_snapshot.py @@ -0,0 +1,181 @@ +"""Sequential full/targeted Fedresurs refresh with atomic snapshot publication.""" + +from __future__ import annotations + +import time +from datetime import date +from uuid import UUID + +from apps.parsers.fedresurs_browser import lookup_company, public_browser +from apps.parsers.fedresurs_company_profile import FedresursUpstreamError +from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord +from apps.parsers.registry_snapshots import ( + SnapshotValidationError, + publish_snapshot, + reject_snapshot, + stage_records, +) +from django.conf import settings +from django.utils import timezone +from organizations.models import Organization + +SOURCE = "fedresurs_company_profile" +RECORD_TYPE = "company_profile" +MAX_TARGETED_ORGANIZATIONS = 500 + + +def _active_organizations(mode: str, organization_uids: list[str] | None): + if mode not in ("full", "targeted"): + raise SnapshotValidationError("invalid_refresh_mode") + queryset = Organization.objects.filter( + opk_registry_membership=True, + directory_imported_at__isnull=False, + ).order_by("uid") + if mode == "targeted": + if not organization_uids or len(organization_uids) > MAX_TARGETED_ORGANIZATIONS: + raise SnapshotValidationError("invalid_targeted_organization_count") + try: + uids = {UUID(str(uid)) for uid in organization_uids} + except (TypeError, ValueError) as exc: + raise SnapshotValidationError("invalid_organization_uid") from exc + if len(uids) != len(organization_uids): + raise SnapshotValidationError("duplicate_organization_uid") + queryset = queryset.filter(uid__in=uids) + if queryset.count() != len(uids): + raise SnapshotValidationError("target_outside_active_directory") + elif organization_uids: + raise SnapshotValidationError("target_not_allowed_for_full_refresh") + return queryset + + +def _lookup_with_retries(driver, organization): + retries = max(0, min(int(settings.FEDRESURS_MAX_RETRIES), 5)) + for attempt in range(retries + 1): + try: + return lookup_company(driver, inn=organization.inn, ogrn=organization.ogrn) + except FedresursUpstreamError as exc: + if ( + str(exc) not in ("upstream_timeout", "browser_failure") + or attempt == retries + ): + raise + time.sleep(min(30, 2**attempt)) + raise AssertionError("unreachable") + + +def refresh_fedresurs_company_profiles( + *, + load_batch: int, + uploaded_by_id: int | None = None, + mode: str = "full", + organization_uids: list[str] | None = None, + on_publish=None, + on_progress=None, +): + """Publish only after all active organizations avoid technical failure.""" + if not settings.FEDRESURS_COMPANY_PROFILE_ENABLED: + raise SnapshotValidationError("fedresurs_source_disabled") + organizations = list(_active_organizations(mode, organization_uids)) + if not organizations: + raise SnapshotValidationError("empty_active_directory") + artifact = ParserSourceArtifact.objects.create( + source=SOURCE, + load_batch=load_batch, + uploaded_by_id=uploaded_by_id, + source_published_at=date.today(), + version="public-ui/v1", + content_type="text/html", + metadata={"mode": mode, "candidate_organizations_count": len(organizations)}, + ) + queried = found = quarantined = 0 + pending = [] + preserved = [] + try: + with public_browser() as driver: + for row_number, organization in enumerate(organizations, 1): + coverage, profile = ( + _lookup_with_retries(driver, organization) + if organization.inn and organization.ogrn + else ("incomplete_identifiers", None) + ) + queried += 1 + if coverage == "match" and profile is not None: + found += 1 + normalized = { + "title": profile["profile"]["full_name"], + "record_date": profile["profile"]["registration_date"] or "", + "status": "active" + if profile["profile"]["status"]["is_active"] + else "inactive", + "url": profile["source_url"], + "payload": { + **profile, + "short_name": organization.name, + "kpp": profile["profile"].get("kpp"), + "status": profile["profile"]["status"], + "registration_date": profile["profile"][ + "registration_date" + ], + "captured_at": timezone.now().isoformat(), + "has_bankruptcy_information": profile["sections"][ + "bankruptcy" + ] + == "loaded", + }, + } + disposition = ParserStagedRecord.Disposition.STAGED + reason = "" + else: + quarantined += 1 + normalized = {} + disposition = ParserStagedRecord.Disposition.QUARANTINED + reason = coverage + if coverage == "ambiguous": + preserved.append(organization.uid) + pending.append( + ParserStagedRecord( + artifact=artifact, + row_number=row_number, + external_id=( + profile["fedresurs_guid"] + if profile is not None + else str(organization.uid) + ), + record_type=RECORD_TYPE, + organization=organization, + normalized_data=normalized, + disposition=disposition, + reason=reason, + ) + ) + if len(pending) >= 500: + stage_records(pending) + if on_progress is not None: + on_progress(queried, len(organizations), found, quarantined) + if row_number < len(organizations): + time.sleep(max(0, float(settings.FEDRESURS_MIN_INTERVAL_SECONDS))) + stage_records(pending) + artifact.parsed_count = queried + artifact.quarantined_count = quarantined + artifact.metadata = { + **artifact.metadata, + "queried_organizations_count": queried, + "matched_organizations_count": found, + "not_matched_organizations_count": quarantined, + } + artifact.status = ParserSourceArtifact.Status.PARSED + artifact.save() + scope_ids = ( + [organization.uid for organization in organizations] + if mode == "targeted" + else None + ) + return artifact, publish_snapshot( + artifact, + on_publish=on_publish, + scope_organization_ids=scope_ids, + preserve_organization_ids=preserved, + ) + except Exception as exc: + reject_snapshot(artifact, exc) + raise diff --git a/src/apps/parsers/integrated_structure_workbook.py b/src/apps/parsers/integrated_structure_workbook.py new file mode 100644 index 0000000..7c4b57b --- /dev/null +++ b/src/apps/parsers/integrated_structure_workbook.py @@ -0,0 +1,223 @@ +"""Validate the two-sheet integrated-structures workbook without database writes.""" + +from __future__ import annotations + +import zipfile +from dataclasses import dataclass, field +from datetime import date, datetime +from io import BytesIO +from typing import BinaryIO +from urllib.parse import urlparse + +from openpyxl import load_workbook + +MAX_WORKBOOK_BYTES = 20 * 1024 * 1024 +MAX_UNCOMPRESSED_BYTES = 100 * 1024 * 1024 +MAX_STAGES = 100_000 +MAX_DOCUMENT_LINKS = 100_000 +STAGE_HEADERS = ("okpo", "stage_id", "stage_date", "stage_prim") +DOCUMENT_HEADERS = ( + "okpo", + "stage_id", + "stage_date", + "doc_id", + "doc_name", + "doc_type", + "doc_n", + "doc_date", + "dsp", + "have_text", + "cons_local", + "cons_web", +) + + +class IntegratedWorkbookError(ValueError): + """Invalid or unsafe integrated-structures upload.""" + + +@dataclass +class ParsedDocument: + external_id: str + name: str + document_type: str + number: str + date: str | None + dsp: bool + have_text: bool + consultant_web_url: str | None + + +@dataclass +class ParsedEvent: + external_id: str + date: str | None + note: str | None + documents: list[str] = field(default_factory=list) + + +@dataclass +class ParsedStructure: + okpo: str + events: list[ParsedEvent] = field(default_factory=list) + + +@dataclass +class ParsedWorkbook: + structures: dict[str, ParsedStructure] + documents: dict[str, ParsedDocument] + links_count: int + + @property + def counts(self) -> tuple[int, int, int, int, int]: + events = [event for item in self.structures.values() for event in item.events] + return ( + len(self.structures), + len(events), + self.links_count, + len(self.documents), + sum(not event.documents for event in events), + ) + + +def _identifier(value: object, *, field_name: str, text_only: bool = False) -> str: + if isinstance(value, str): + value = value.strip() + if value and len(value) <= 64 and value.isdecimal(): + return value + elif not text_only and isinstance(value, int) and not isinstance(value, bool): + if value >= 0: + return str(value) + raise IntegratedWorkbookError(f"invalid_{field_name}") + + +def _date(value: object, *, field_name: str) -> str | None: + if value is None or value == "": + return None + if isinstance(value, datetime): + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + if isinstance(value, str): + try: + return date.fromisoformat(value.strip()).isoformat() + except ValueError: + pass + raise IntegratedWorkbookError(f"invalid_{field_name}") + + +def _text(value: object, *, required: bool = False, max_length: int = 1024) -> str: + result = str(value).strip() if value is not None else "" + if len(result) > max_length or (required and not result): + raise IntegratedWorkbookError("invalid_text") + return result + + +def _flag(value: object, *, field_name: str) -> bool: + if isinstance(value, int) and not isinstance(value, bool) and value in (0, 1): + return bool(value) + raise IntegratedWorkbookError(f"invalid_{field_name}") + + +def _consultant_url(value: object) -> str | None: + url = _text(value, max_length=2048) + if not url: + return None + parsed = urlparse(url) + if ( + parsed.scheme != "https" + or parsed.hostname != "online.consultant.ru" + or parsed.username is not None + or parsed.password is not None + or parsed.port not in (None, 443) + ): + raise IntegratedWorkbookError("invalid_consultant_url") + return url + + +def _sheet_rows(workbook, sheet_name: str, headers: tuple[str, ...], limit: int): + if sheet_name not in workbook.sheetnames: + raise IntegratedWorkbookError(f"missing_{sheet_name}") + sheet = workbook[sheet_name] + iterator = sheet.iter_rows(values_only=True) + actual = next(iterator, None) + if actual is None or tuple(actual[: len(headers)]) != headers: + raise IntegratedWorkbookError(f"invalid_{sheet_name}_headers") + for row_number, row in enumerate(iterator, 2): + if row_number > limit + 1: + raise IntegratedWorkbookError(f"too_many_{sheet_name}_rows") + if any(value is not None for value in row): + yield row_number, row + + +def parse_integrated_workbook(handle: BinaryIO) -> ParsedWorkbook: # noqa: C901 + """Parse bounded XLSX input, preserving text-formatted OKPO identifiers.""" + raw = handle.read(MAX_WORKBOOK_BYTES + 1) + if not raw or len(raw) > MAX_WORKBOOK_BYTES: + raise IntegratedWorkbookError("invalid_workbook_size") + try: + with zipfile.ZipFile(BytesIO(raw)) as archive: + if ( + sum(item.file_size for item in archive.infolist()) + > MAX_UNCOMPRESSED_BYTES + ): + raise IntegratedWorkbookError("workbook_expanded_size_exceeded") + workbook = load_workbook(BytesIO(raw), read_only=True, data_only=True) + except (zipfile.BadZipFile, OSError, ValueError) as exc: + if isinstance(exc, IntegratedWorkbookError): + raise + raise IntegratedWorkbookError("invalid_xlsx") from exc + try: + structures: dict[str, ParsedStructure] = {} + events: dict[tuple[str, str], ParsedEvent] = {} + for _, row in _sheet_rows(workbook, "is_stages", STAGE_HEADERS, MAX_STAGES): + okpo = _identifier(row[0], field_name="okpo", text_only=True) + stage_id = _identifier(row[1], field_name="stage_id") + key = (okpo, stage_id) + if key in events: + raise IntegratedWorkbookError("duplicate_stage") + event = ParsedEvent( + external_id=stage_id, + date=_date(row[2], field_name="stage_date"), + note=_text(row[3]) or None, + ) + structures.setdefault(okpo, ParsedStructure(okpo)).events.append(event) + events[key] = event + if not structures: + raise IntegratedWorkbookError("empty_stages") + + documents: dict[str, ParsedDocument] = {} + links: set[tuple[str, str, str]] = set() + for _, row in _sheet_rows( + workbook, "is_stages_docs", DOCUMENT_HEADERS, MAX_DOCUMENT_LINKS + ): + okpo = _identifier(row[0], field_name="okpo", text_only=True) + stage_id = _identifier(row[1], field_name="stage_id") + document_id = _identifier(row[3], field_name="doc_id") + event = events.get((okpo, stage_id)) + if event is None: + raise IntegratedWorkbookError("unknown_stage") + if _date(row[2], field_name="stage_date") != event.date: + raise IntegratedWorkbookError("stage_date_mismatch") + document = ParsedDocument( + external_id=document_id, + name=_text(row[4], required=True), + document_type=_text(row[5], max_length=255), + number=_text(row[6], max_length=255), + date=_date(row[7], field_name="doc_date"), + dsp=_flag(row[8], field_name="dsp"), + have_text=_flag(row[9], field_name="have_text"), + consultant_web_url=_consultant_url(row[11]), + ) + # cons_local is provenance only and is intentionally never exposed. + previous = documents.setdefault(document_id, document) + if previous != document: + raise IntegratedWorkbookError("conflicting_document_metadata") + link = (okpo, stage_id, document_id) + if link in links: + raise IntegratedWorkbookError("duplicate_document_link") + links.add(link) + event.documents.append(document_id) + return ParsedWorkbook(structures, documents, len(links)) + finally: + workbook.close() diff --git a/src/apps/parsers/integrated_structures.py b/src/apps/parsers/integrated_structures.py new file mode 100644 index 0000000..148ec93 --- /dev/null +++ b/src/apps/parsers/integrated_structures.py @@ -0,0 +1,306 @@ +"""Atomic XLSX/RTF import for integrated structures.""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Iterable +from dataclasses import dataclass +from io import BytesIO +from typing import BinaryIO + +from apps.parsers.integrated_structure_workbook import ( + IntegratedWorkbookError, + ParsedWorkbook, + parse_integrated_workbook, +) +from apps.parsers.models import ( + IntegratedStructureAttachment, + IntegratedStructureDocument, + IntegratedStructureEvent, + IntegratedStructureEventDocument, + ParserLoadLog, + ParserSourceArtifact, + ParserStagedRecord, +) +from apps.parsers.registry_snapshots import ( + publish_snapshot, + reject_snapshot, + save_artifact_file, + stable_registry_uid, + stage_records, +) +from apps.parsers.services import ParserLoadLogService +from django.core.files.base import ContentFile +from organizations.models import Organization, OrganizationSourceRecord + +PARSER_SOURCE = "integrated_structure_documents" +RECORD_SOURCE = "integrated_structures" +RECORD_TYPE = "integrated_structure" +RTF_NAME = re.compile(r"^([0-9]{1,12})\.rtf$", re.IGNORECASE) +MAX_ATTACHMENTS = 500 +MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024 +MAX_TOTAL_ATTACHMENT_BYTES = 100 * 1024 * 1024 + + +@dataclass(frozen=True) +class ValidatedAttachment: + document_id: str + name: str + sha256: str + data: bytes + + +def _resolve_attachment_document_id(raw_id: str, parsed: ParsedWorkbook) -> str: + if raw_id in parsed.documents: + return raw_id + candidates = [ + candidate for candidate in parsed.documents if int(candidate) == int(raw_id) + ] + if len(candidates) != 1: + raise IntegratedWorkbookError("unknown_or_ambiguous_attachment_document") + return candidates[0] + + +def _validated_attachments( + files: Iterable[BinaryIO], parsed: ParsedWorkbook +) -> dict[str, ValidatedAttachment]: + result: dict[str, ValidatedAttachment] = {} + total_size = 0 + for count, file in enumerate(files, 1): + if count > MAX_ATTACHMENTS: + raise IntegratedWorkbookError("too_many_attachments") + name = str(getattr(file, "name", "")) + match = RTF_NAME.fullmatch(name) + if match is None: + raise IntegratedWorkbookError("invalid_attachment_name") + document_id = _resolve_attachment_document_id(match[1], parsed) + if document_id in result: + raise IntegratedWorkbookError("duplicate_attachment") + if document_id not in parsed.documents: + raise IntegratedWorkbookError("unknown_attachment_document") + file.seek(0) + data = file.read(MAX_ATTACHMENT_BYTES + 1) + if len(data) > MAX_ATTACHMENT_BYTES: + raise IntegratedWorkbookError("attachment_too_large") + total_size += len(data) + if total_size > MAX_TOTAL_ATTACHMENT_BYTES: + raise IntegratedWorkbookError("attachments_too_large") + if not data.startswith(b"{\\rtf"): + raise IntegratedWorkbookError("invalid_rtf_signature") + result[document_id] = ValidatedAttachment( + document_id=document_id, + name=name, + sha256=hashlib.sha256(data).hexdigest(), + data=data, + ) + return result + + +def _composite_hash( + workbook: bytes, attachments: dict[str, ValidatedAttachment] +) -> str: + digest = hashlib.sha256() + digest.update(len(workbook).to_bytes(8, "big")) + digest.update(workbook) + for document_id, attachment in sorted(attachments.items()): + digest.update(document_id.encode("ascii")) + digest.update(attachment.sha256.encode("ascii")) + return digest.hexdigest() + + +def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]: + by_okpo: dict[str, Organization] = {} + for organization in Organization.objects.filter( + okpo__in=parsed.structures, directory_imported_at__isnull=False + ).only("uid", "name", "inn", "ogrn", "kpp", "okpo"): + if organization.okpo in by_okpo: + raise IntegratedWorkbookError("ambiguous_canonical_okpo") + by_okpo[organization.okpo] = organization + if len(by_okpo) != len(parsed.structures): + raise IntegratedWorkbookError("unknown_canonical_okpo") + return by_okpo + + +def _save_attachments( + attachments: dict[str, ValidatedAttachment], +) -> dict[str, IntegratedStructureAttachment]: + result = {} + for document_id, attachment in attachments.items(): + stored = IntegratedStructureAttachment.objects.filter( + sha256=attachment.sha256 + ).first() + if stored is None: + stored = IntegratedStructureAttachment( + sha256=attachment.sha256, size_bytes=len(attachment.data) + ) + stored.file.save(attachment.name, ContentFile(attachment.data), save=False) + stored.save() + result[document_id] = stored + return result + + +def _publish_relations( + artifact: ParserSourceArtifact, + parsed: ParsedWorkbook, + attachments: dict[str, ValidatedAttachment], +) -> None: + stored_attachments = _save_attachments(attachments) + documents = {} + for document_id, document in parsed.documents.items(): + documents[document_id] = IntegratedStructureDocument.objects.create( + artifact=artifact, + external_id=document_id, + name=document.name, + document_type=document.document_type, + number=document.number, + document_date=document.date, + dsp=document.dsp, + have_text=document.have_text, + consultant_web_url=document.consultant_web_url or "", + attachment_filename=attachments[document_id].name + if document_id in attachments + else "", + attachment=stored_attachments.get(document_id), + ) + for okpo, structure in parsed.structures.items(): + external_id = f"integrated-structure:{okpo}" + record = OrganizationSourceRecord.objects.get( + uid=stable_registry_uid(RECORD_SOURCE, external_id) + ) + record.integrated_events.all().delete() + for item in structure.events: + event = IntegratedStructureEvent.objects.create( + record=record, + external_id=item.external_id, + event_date=item.date, + note=item.note or "", + ) + IntegratedStructureEventDocument.objects.bulk_create( + [ + IntegratedStructureEventDocument( + event=event, document=documents[document_id] + ) + for document_id in item.documents + ] + ) + + +def import_integrated_structures( + *, + workbook: BinaryIO, + original_name: str, + attachments: Iterable[BinaryIO], + uploaded_by_id: int | None = None, +) -> tuple[ParserSourceArtifact, dict]: + """Validate all inputs, stage all structures, then publish one transaction.""" + workbook.seek(0) + workbook_bytes = workbook.read(20 * 1024 * 1024 + 1) + parsed = parse_integrated_workbook(BytesIO(workbook_bytes)) + validated_attachments = _validated_attachments(attachments, parsed) + composite_hash = _composite_hash(workbook_bytes, validated_attachments) + existing = ( + ParserSourceArtifact.objects.filter( + source=PARSER_SOURCE, + status=ParserSourceArtifact.Status.PUBLISHED, + metadata__composite_sha256=composite_hash, + ) + .order_by("-created_at") + .first() + ) + if existing is not None: + return existing, { + "status": "skipped", + "reason": "identical_batch", + "batch_id": existing.load_batch, + "records_count": existing.published_count, + **existing.metadata, + } + organizations = _resolve_organizations(parsed) + load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( + source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS + ) + artifact = ParserSourceArtifact.objects.create( + source=PARSER_SOURCE, + original_name=original_name[:512], + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + load_batch=batch_id, + uploaded_by_id=uploaded_by_id, + version="is_stages+is_stages_docs/v1", + parsed_count=len(parsed.structures), + metadata={ + "composite_sha256": composite_hash, + "structures_count": parsed.counts[0], + "event_rows_count": parsed.counts[1], + "document_link_rows_count": parsed.counts[2], + "unique_documents_count": parsed.counts[3], + "events_without_documents_count": parsed.counts[4], + "attachments_count": len(validated_attachments), + "missing_attachments_count": len(parsed.documents) + - len(validated_attachments), + }, + ) + try: + save_artifact_file(artifact, BytesIO(workbook_bytes)) + pending = [] + for index, (okpo, structure) in enumerate(parsed.structures.items(), 1): + organization = organizations[okpo] + external_id = f"integrated-structure:{okpo}" + document_ids = { + doc_id for event in structure.events for doc_id in event.documents + } + event_dates = [event.date for event in structure.events if event.date] + pending.append( + ParserStagedRecord( + artifact=artifact, + row_number=index, + external_id=external_id, + record_type=RECORD_TYPE, + organization=organization, + normalized_data={ + "title": organization.name, + "record_date": max(event_dates) if event_dates else "", + "status": "active", + "payload": { + "structure": { + "events_count": len(structure.events), + "document_links_count": sum( + len(event.documents) for event in structure.events + ), + "unique_documents_count": len(document_ids), + "first_event_date": min(event_dates) if event_dates else "", + "latest_event_date": max(event_dates) if event_dates else "", + }, + }, + }, + ) + ) + stage_records(pending) + artifact.status = ParserSourceArtifact.Status.PARSED + artifact.save(update_fields=["status", "updated_at"]) + + def finalize(artifact, result): + _publish_relations(artifact, parsed, validated_attachments) + ParserLoadLogService.update( + load_log, + status=ParserLoadLog.Status.SUCCESS, + records_count=result.published, + ) + + result = publish_snapshot( + artifact, record_source=RECORD_SOURCE, on_publish=finalize + ) + return artifact, { + "status": "published", + "batch_id": batch_id, + "records_count": result.published, + **artifact.metadata, + } + except Exception as exc: + reject_snapshot(artifact, exc) + ParserLoadLogService.update( + load_log, + status=ParserLoadLog.Status.FAILED, + error_message=type(exc).__name__, + ) + raise diff --git a/src/apps/parsers/migrations/0038_auto_20260924_1056.py b/src/apps/parsers/migrations/0038_auto_20260924_1056.py new file mode 100644 index 0000000..2e9db7b --- /dev/null +++ b/src/apps/parsers/migrations/0038_auto_20260924_1056.py @@ -0,0 +1,112 @@ +# Generated by Django 3.2.25 on 2026-09-24 10:56 + +import apps.parsers.artifact_storage +import apps.parsers.models +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('organizations', '0014_auto_20260924_1056'), + ('parsers', '0037_private_sro_artifact_storage'), + ] + + operations = [ + migrations.CreateModel( + name='IntegratedStructureAttachment', + fields=[ + ('sha256', models.CharField(max_length=64, primary_key=True, serialize=False)), + ('size_bytes', models.PositiveIntegerField()), + ('file', models.FileField(max_length=1024, storage=apps.parsers.artifact_storage.ParserArtifactStorage(), upload_to=apps.parsers.models.integrated_attachment_upload_to)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ], + options={ + 'db_table': 'parsers_integrated_structure_attachment', + }, + ), + migrations.CreateModel( + name='IntegratedStructureDocument', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('external_id', models.CharField(max_length=64)), + ('name', models.CharField(max_length=1024)), + ('document_type', models.CharField(blank=True, max_length=255)), + ('number', models.CharField(blank=True, max_length=255)), + ('document_date', models.DateField(blank=True, null=True)), + ('dsp', models.BooleanField(default=False)), + ('have_text', models.BooleanField(default=False)), + ('consultant_web_url', models.URLField(blank=True, max_length=2048)), + ], + options={ + 'db_table': 'parsers_integrated_structure_document', + }, + ), + migrations.CreateModel( + name='IntegratedStructureEvent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('external_id', models.CharField(max_length=64)), + ('event_date', models.DateField(blank=True, null=True)), + ('note', models.TextField(blank=True)), + ('record', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='integrated_events', to='organizations.organizationsourcerecord')), + ], + options={ + 'db_table': 'parsers_integrated_structure_event', + }, + ), + migrations.AlterField( + model_name='genericparserrecord', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('hh', 'Вакансии HeadHunter'), ('superjob', 'Вакансии SuperJob')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parserbatchsequence', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур')], help_text='Источник данных', max_length=50, unique=True, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parserloadlog', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parsersourceartifact', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур')], db_index=True, max_length=50), + ), + migrations.CreateModel( + name='IntegratedStructureEventDocument', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('document', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='event_links', to='parsers.integratedstructuredocument')), + ('event', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='document_links', to='parsers.integratedstructureevent')), + ], + options={ + 'db_table': 'parsers_integrated_structure_event_document', + }, + ), + migrations.AddField( + model_name='integratedstructuredocument', + name='artifact', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='integrated_documents', to='parsers.parsersourceartifact'), + ), + migrations.AddField( + model_name='integratedstructuredocument', + name='attachment', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='documents', to='parsers.integratedstructureattachment'), + ), + migrations.AddConstraint( + model_name='integratedstructureeventdocument', + constraint=models.UniqueConstraint(fields=('event', 'document'), name='unique_integrated_event_document'), + ), + migrations.AddConstraint( + model_name='integratedstructureevent', + constraint=models.UniqueConstraint(fields=('record', 'external_id'), name='unique_integrated_event_per_structure'), + ), + migrations.AddConstraint( + model_name='integratedstructuredocument', + constraint=models.UniqueConstraint(fields=('artifact', 'external_id'), name='unique_integrated_document_per_batch'), + ), + ] diff --git a/src/apps/parsers/migrations/0039_integratedstructuredocument_attachment_filename.py b/src/apps/parsers/migrations/0039_integratedstructuredocument_attachment_filename.py new file mode 100644 index 0000000..fa8c638 --- /dev/null +++ b/src/apps/parsers/migrations/0039_integratedstructuredocument_attachment_filename.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2026-09-24 11:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0038_auto_20260924_1056'), + ] + + operations = [ + migrations.AddField( + model_name='integratedstructuredocument', + name='attachment_filename', + field=models.CharField(blank=True, max_length=255), + ), + ] diff --git a/src/apps/parsers/migrations/0040_auto_20260924_1114.py b/src/apps/parsers/migrations/0040_auto_20260924_1114.py new file mode 100644 index 0000000..b9b397b --- /dev/null +++ b/src/apps/parsers/migrations/0040_auto_20260924_1114.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.25 on 2026-09-24 11:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('parsers', '0039_integratedstructuredocument_attachment_filename'), + ] + + operations = [ + migrations.AlterField( + model_name='genericparserrecord', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса'), ('hh', 'Вакансии HeadHunter'), ('superjob', 'Вакансии SuperJob')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parserbatchsequence', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса')], help_text='Источник данных', max_length=50, unique=True, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parserloadlog', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса')], db_index=True, help_text='Источник данных', max_length=50, verbose_name='источник'), + ), + migrations.AlterField( + model_name='parsersourceartifact', + name='source', + field=models.CharField(choices=[('industrial', 'Сертификаты промышленного производства'), ('industrial_products', 'Реестр промышленной продукции'), ('manufactures', 'Реестр производителей'), ('inspections', 'Единый реестр проверок'), ('procurements', 'Единая информационная система закупок'), ('fns_reports', 'Бухгалтерская отчетность ФНС'), ('procurements_44fz', 'Закупки 44-ФЗ'), ('procurements_223fz', 'Закупки 223-ФЗ'), ('contracts', 'Контракты ЕИС'), ('unfair_suppliers', 'Недобросовестные поставщики'), ('fas_goz', 'Уклонение от ГОЗ'), ('arbitration', 'Арбитражные дела'), ('fedresurs_bankruptcy', 'Банкротства Федресурс'), ('fstec', 'Реестры ФСТЭК'), ('trudvsem', 'Вакансии Работа России'), ('gosedo_address_directory', 'Глобальный адресный справочник ГосЭДО'), ('media_news', 'Новости СМИ'), ('ropk_sanctions', 'РОПК — Санкции'), ('fns_sme_support_recipients', 'Реестр субъектов МСП — получателей поддержки'), ('budget_ubpandnubp', 'Реестр участников бюджетного процесса'), ('sro_membership_check', 'Проверка членства в СРО'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса')], db_index=True, max_length=50), + ), + ] diff --git a/src/apps/parsers/migrations/0041_fedresurs_disabled_schedule.py b/src/apps/parsers/migrations/0041_fedresurs_disabled_schedule.py new file mode 100644 index 0000000..24c2c86 --- /dev/null +++ b/src/apps/parsers/migrations/0041_fedresurs_disabled_schedule.py @@ -0,0 +1,41 @@ +"""Register daily Fedresurs refresh, disabled until a complete dev live-run succeeds.""" + +from django.db import migrations + + +def seed(apps, schema_editor): + crontab = apps.get_model("django_celery_beat", "CrontabSchedule") + tasks = apps.get_model("django_celery_beat", "PeriodicTask") + schedule, _ = crontab.objects.get_or_create( + minute="30", + hour="2", + day_of_week="*", + day_of_month="*", + month_of_year="*", + timezone="Europe/Moscow", + ) + tasks.objects.update_or_create( + name="parser:fedresurs-company-profile:full-msk", + defaults={ + "task": "parsers.fedresurs_company_profile.refresh", + "crontab": schedule, + "enabled": False, + "args": "[]", + "kwargs": '{"mode":"full"}', + "description": "Enable only after a complete successful public-UI refresh on dev.", + }, + ) + + +def unseed(apps, schema_editor): + apps.get_model("django_celery_beat", "PeriodicTask").objects.filter( + name="parser:fedresurs-company-profile:full-msk" + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0040_auto_20260924_1114"), + ("django_celery_beat", "0018_improve_crontab_helptext"), + ] + operations = [migrations.RunPython(seed, unseed)] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 22ddec8..d100b3f 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -53,6 +53,12 @@ class ParserLoadLog(TimestampMixin, models.Model): _("Реестр участников бюджетного процесса"), ) SRO_MEMBERSHIP_CHECK = "sro_membership_check", _("Проверка членства в СРО") + INTEGRATED_STRUCTURE_DOCUMENTS = ( + "integrated_structure_documents", _("Документы интегрированных структур") + ) + FEDRESURS_COMPANY_PROFILE = ( + "fedresurs_company_profile", _("Карточки компаний Федресурса") + ) class Status(models.TextChoices): SUCCESS = "success", _("Успешно") @@ -1308,3 +1314,118 @@ class ParsingSettings(TimestampMixin, models.Model): def __str__(self) -> str: return "Настройки парсинга" + + +def integrated_attachment_upload_to(instance, filename): + """Private, content-addressed attachment location.""" + return ( + "parser-artifacts/integrated_structure_documents/attachments/" + f"{instance.sha256}.rtf" + ) + + +class IntegratedStructureAttachment(models.Model): + """One immutable RTF payload shared by document references and batches.""" + + sha256 = models.CharField(max_length=64, primary_key=True) + size_bytes = models.PositiveIntegerField() + file = models.FileField( + upload_to=integrated_attachment_upload_to, + storage=ParserArtifactStorage(), + max_length=1024, + ) + created_at = models.DateTimeField(auto_now_add=True) + + class Meta: + db_table = "parsers_integrated_structure_attachment" + + def __str__(self) -> str: + return self.sha256 + + +class IntegratedStructureDocument(models.Model): + """Document metadata scoped to one validated workbook batch.""" + + artifact = models.ForeignKey( + ParserSourceArtifact, on_delete=models.CASCADE, related_name="integrated_documents" + ) + external_id = models.CharField(max_length=64) + name = models.CharField(max_length=1024) + document_type = models.CharField(max_length=255, blank=True) + number = models.CharField(max_length=255, blank=True) + document_date = models.DateField(null=True, blank=True) + dsp = models.BooleanField(default=False) + have_text = models.BooleanField(default=False) + consultant_web_url = models.URLField(max_length=2048, blank=True) + attachment_filename = models.CharField(max_length=255, blank=True) + attachment = models.ForeignKey( + IntegratedStructureAttachment, + on_delete=models.PROTECT, + null=True, + blank=True, + related_name="documents", + ) + + class Meta: + db_table = "parsers_integrated_structure_document" + constraints = [ + models.UniqueConstraint( + fields=["artifact", "external_id"], + name="unique_integrated_document_per_batch", + ) + ] + + def __str__(self) -> str: + return self.name + + +class IntegratedStructureEvent(models.Model): + """An event of a published structure, never a public source record itself.""" + + record = models.ForeignKey( + "organizations.OrganizationSourceRecord", + on_delete=models.CASCADE, + related_name="integrated_events", + ) + external_id = models.CharField(max_length=64) + event_date = models.DateField(null=True, blank=True) + note = models.TextField(blank=True) + + class Meta: + db_table = "parsers_integrated_structure_event" + constraints = [ + models.UniqueConstraint( + fields=["record", "external_id"], + name="unique_integrated_event_per_structure", + ) + ] + + def __str__(self) -> str: + return self.external_id + + +class IntegratedStructureEventDocument(models.Model): + """Many-to-many relation preserving events without documents.""" + + event = models.ForeignKey( + IntegratedStructureEvent, + on_delete=models.CASCADE, + related_name="document_links", + ) + document = models.ForeignKey( + IntegratedStructureDocument, + on_delete=models.PROTECT, + related_name="event_links", + ) + + class Meta: + db_table = "parsers_integrated_structure_event_document" + constraints = [ + models.UniqueConstraint( + fields=["event", "document"], + name="unique_integrated_event_document", + ) + ] + + def __str__(self) -> str: + return f"{self.event_id}:{self.document_id}" diff --git a/src/apps/parsers/registry_snapshots.py b/src/apps/parsers/registry_snapshots.py index f413993..d62b33d 100644 --- a/src/apps/parsers/registry_snapshots.py +++ b/src/apps/parsers/registry_snapshots.py @@ -200,6 +200,7 @@ def _set_published_count( def publish_snapshot( artifact: ParserSourceArtifact, *, + record_source: str | None = None, on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None, scope_organization_ids: list[uuid.UUID] | None = None, preserve_organization_ids: list[uuid.UUID] | None = None, @@ -209,7 +210,7 @@ def publish_snapshot( Preservation affects only obsolete deletion for the selected organizations. Valid staged rows are still upserted, including their existing stable IDs. """ - source = artifact.source + source = record_source or artifact.source descriptor = get_source_group_descriptor(source) staged = ParserStagedRecord.objects.filter( artifact=artifact, disposition=ParserStagedRecord.Disposition.STAGED diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index 10dc71e..be227c8 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -757,6 +757,20 @@ class ParserRunResponseSerializer(serializers.Serializer): ) +class IntegratedStructureUploadResponseSerializer(serializers.Serializer): + status = serializers.ChoiceField(choices=("published", "skipped")) + artifact_id = serializers.UUIDField() + task_id = serializers.CharField() + batch_id = serializers.IntegerField() + records_count = serializers.IntegerField() + structures_count = serializers.IntegerField() + event_rows_count = serializers.IntegerField() + document_link_rows_count = serializers.IntegerField() + unique_documents_count = serializers.IntegerField() + events_without_documents_count = serializers.IntegerField() + reason = serializers.CharField(required=False) + + # ============================================================================= # Служебные модели # ============================================================================= @@ -1150,6 +1164,28 @@ class SroParserRunRequestSerializer(SourceCardRefreshRequestSerializer): return super().to_internal_value(data) +class FedresursParserRunRequestSerializer(SourceCardRefreshRequestSerializer): + mode = serializers.ChoiceField( + choices=("full", "targeted"), required=False, default="full" + ) + organization_uids = serializers.ListField( + child=serializers.UUIDField(), required=False, max_length=500 + ) + + def validate(self, attrs): + mode = attrs.get("mode", "full") + uids = attrs.get("organization_uids") + if (mode == "targeted") != bool(uids): + raise serializers.ValidationError( + {"organization_uids": "Required only for targeted refresh."} + ) + attrs["params"] = { + **attrs.get("params", {}), + **({"organization_uids": [str(uid) for uid in uids]} if uids else {}), + } + return attrs + + class SourceCardRefreshTaskSerializer(serializers.Serializer): """Queued task info after refresh start.""" diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index ec76783..f655948 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -49,11 +49,13 @@ SNAPSHOT_CARD_SLUGS = { "budget_ubpandnubp": "budget-process-registry", "fns_sme_support_recipients": "sme-support-recipients-registry", "sro_membership_check": "sro-membership-check", + "fedresurs_company_profile": "fedresurs-company-profile", } SNAPSHOT_SOURCE_GROUPS = ( "budget_process_registry", "government_support", "sro_membership", + "fedresurs_company_profile", ) @@ -466,6 +468,41 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( ), ), ), + SourceCardDefinition( + slug="integrated-structure-documents", + title="Документы интегрированных структур", + description="События интегрированных структур и связанные документы.", + order=150, + task_names=("parsers.integrated_structure_documents.import",), + source_items=( + SourceItemDefinition( + code="integrated_structures", + title="Интегрированные структуры", + description="Импорт XLSX и RTF-документов.", + parser_source=ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS, + refresh_key="integrated_structure_documents", + ), + ), + supports_refresh=False, + upload_url="/api/v1/parsers/upload/integrated_structure_documents/", + ), + SourceCardDefinition( + slug="fedresurs-company-profile", + title="Федресурс: сведения о юридических лицах", + description="Публичные карточки организаций Федресурса.", + order=160, + task_names=("parsers.fedresurs_company_profile.refresh",), + source_items=( + SourceItemDefinition( + code="fedresurs_company_profile", + title="Сведения о юридических лицах Федресурса", + description="Публичные карточки организаций.", + parser_source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, + refresh_key="fedresurs_company_profile", + ), + ), + refresh_interval=timedelta(days=1), + ), ) SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS} @@ -501,6 +538,7 @@ SOURCE_RECORD_SOURCES_BY_ITEM_CODE["procurements"] = [ ParserLoadLog.Source.PROCUREMENTS_44FZ, ParserLoadLog.Source.PROCUREMENTS_223FZ, ] +SOURCE_RECORD_SOURCES_BY_ITEM_CODE["integrated_structures"] = ["integrated_structures"] class SourceCardService: @@ -678,7 +716,7 @@ class SourceCardService: } @classmethod - def refresh_card( + def refresh_card( # noqa: C901 cls, *, slug: str, @@ -691,9 +729,65 @@ class SourceCardService: raise ValidationError( {"detail": "Обновление для карточки не поддерживается."} ) - params = cls._validate_refresh_params(definition, params or {}) + if ( + slug == "fedresurs-company-profile" + and not settings.FEDRESURS_COMPANY_PROFILE_ENABLED + ): + raise APIValidationError( + message="Источник Федресурс выключен до успешной проверки на dev.", + code="fedresurs_source_disabled", + ) + params = dict(params or {}) + if slug == "fedresurs-company-profile": + from uuid import UUID + + mode = mode or params.pop("mode", "full") + organization_uids = params.pop("organization_uids", None) + if params or mode not in ("full", "targeted"): + raise APIValidationError( + message="Недопустимые параметры обновления.", + code="invalid_refresh_params", + ) + if mode == "targeted": + if ( + not isinstance(organization_uids, list) + or not 1 <= len(organization_uids) <= 500 + ): + raise APIValidationError( + message="Укажите от 1 до 500 организаций.", + code="invalid_targeted_organization_count", + ) + try: + organization_uids = [ + str(UUID(str(uid))) for uid in organization_uids + ] + except (TypeError, ValueError) as exc: + raise APIValidationError( + message="Недопустимый UID организации.", + code="invalid_organization_uid", + ) from exc + if len(set(organization_uids)) != len(organization_uids): + raise APIValidationError( + message="UID организации повторяется.", + code="duplicate_organization_uid", + ) + params = {"organization_uids": organization_uids} + elif organization_uids is not None: + raise APIValidationError( + message="Список организаций допустим только в targeted.", + code="invalid_targeted_organization_count", + ) + else: + params = cls._validate_refresh_params(definition, params) if mode is not None: - if slug != "sro-membership-check" or mode not in ("incremental", "full"): + allowed_modes = ( + ("incremental", "full") + if slug == "sro-membership-check" + else ("full", "targeted") + if slug == "fedresurs-company-profile" + else () + ) + if mode not in allowed_modes: raise APIValidationError( message="Недопустимый режим обновления.", code="invalid_mode" ) @@ -1152,7 +1246,7 @@ class SourceCardService: return validated @classmethod - def _launch_refresh( + def _launch_refresh( # noqa: C901 cls, definition: SourceCardDefinition, *, @@ -1224,6 +1318,7 @@ class SourceCardService: elif definition.slug in SNAPSHOT_CARD_SLUGS.values(): from apps.parsers.tasks_registry_snapshots import ( parse_budget_registry, + parse_fedresurs_company_profiles, parse_sme_support_recipients, parse_sro_membership, ) @@ -1246,6 +1341,17 @@ class SourceCardService: "sro_membership_check", ), ) + elif definition.slug == "fedresurs-company-profile": + kwargs["mode"] = params.get("mode", "full") + if "organization_uids" in params: + kwargs["organization_uids"] = params["organization_uids"] + task_specs = ( + ( + parse_fedresurs_company_profiles, + "parsers.fedresurs_company_profile.refresh", + "fedresurs_company_profile", + ), + ) else: task_specs = ( ( diff --git a/src/apps/parsers/source_registry.py b/src/apps/parsers/source_registry.py index f506788..2c20bae 100644 --- a/src/apps/parsers/source_registry.py +++ b/src/apps/parsers/source_registry.py @@ -56,6 +56,34 @@ class ParserSourceDescriptor: PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { + "fedresurs_company_profile": ParserSourceDescriptor( + key="fedresurs_company_profile", + source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, + title="Федресурс: сведения о юридических лицах", + agency="Федресурс", + data_scope="Публичные карточки организаций канонического реестра", + task_name="parsers.fedresurs_company_profile.refresh", + mode="public_browser", + upstream_url="https://fedresurs.ru/", + access_method="public_selenium", + parser_strategy="exact_identity_sequential_snapshot", + admin_only=True, + ), + "integrated_structure_documents": ParserSourceDescriptor( + key="integrated_structure_documents", + source=ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS, + title="Документы интегрированных структур", + agency="Реестр интегрированных структур", + data_scope="События и документы интегрированных структур", + task_name="parsers.integrated_structure_documents.import", + mode="manual_upload", + access_method="xlsx_rtf_upload", + parser_strategy="atomic_two_sheet_import", + supports_file_upload=True, + supports_refresh=False, + admin_only=True, + upload_route="parsers/upload/integrated_structure_documents", + ), "sro_membership_check": ParserSourceDescriptor( key="sro_membership_check", source=ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK, diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index c289d66..49214bf 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -74,6 +74,7 @@ from apps.parsers.source_artifacts import cleanup_parser_source_artifacts from apps.parsers.source_registry import PARSER_SOURCES from apps.parsers.tasks_registry_snapshots import ( # noqa: F401 parse_budget_registry, + parse_fedresurs_company_profiles, parse_sme_support_recipients, parse_sro_membership, ) diff --git a/src/apps/parsers/tasks_registry_snapshots.py b/src/apps/parsers/tasks_registry_snapshots.py index 6dbd4ac..477b34a 100644 --- a/src/apps/parsers/tasks_registry_snapshots.py +++ b/src/apps/parsers/tasks_registry_snapshots.py @@ -79,7 +79,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict: def _claim_snapshot_job(job, source, task_name): with transaction.atomic(): - if source == "sro_membership_check": + if source in ("sro_membership_check", "fedresurs_company_profile"): sequence, _ = ParserBatchSequence.objects.get_or_create(source=source) ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk) if ( @@ -261,3 +261,29 @@ def parse_sro_membership( track_progress=True, mode=mode, ) + + +@shared_task( + bind=True, + name="parsers.fedresurs_company_profile.refresh", + soft_time_limit=24 * 3600, + time_limit=24 * 3600 + 300, +) +def parse_fedresurs_company_profiles( + self, *, requested_by_id: int | None = None, mode: str = "full", + organization_uids: list[str] | None = None, +) -> dict: + from apps.parsers.fedresurs_snapshot import refresh_fedresurs_company_profiles + + return _run_snapshot( + self, + source="fedresurs_company_profile", + refresh=partial( + refresh_fedresurs_company_profiles, + mode=mode, + organization_uids=organization_uids, + ), + requested_by_id=requested_by_id, + track_progress=True, + mode=mode, + ) diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index c2ba659..8979efc 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -38,6 +38,7 @@ from apps.parsers.ropk_sanctions import ( ROPK_SANCTIONS_MIME, ) from apps.parsers.serializers import ( + FedresursParserRunRequestSerializer, FinancialReportDetailSerializer, FinancialReportSerializer, FNSFileUploadSerializer, @@ -145,6 +146,7 @@ SOURCE_RESULT_TAGS = { } TASKS_BY_NAME = { + "parsers.fedresurs_company_profile.refresh": tasks.parse_fedresurs_company_profiles, "apps.parsers.tasks.parse_industrial_production": tasks.parse_industrial_production, "apps.parsers.tasks.parse_industrial_products": tasks.parse_industrial_products, "apps.parsers.tasks.parse_manufactures": tasks.parse_manufactures, @@ -403,6 +405,20 @@ ROPK_SANCTIONS_UPLOAD_FILE_PARAM = openapi.Parameter( type=openapi.TYPE_FILE, required=True, ) +INTEGRATED_WORKBOOK_PARAM = openapi.Parameter( + "workbook", + openapi.IN_FORM, + description="Один XLSX с листами is_stages и is_stages_docs, до 20 МиБ", + type=openapi.TYPE_FILE, + required=True, +) +INTEGRATED_ATTACHMENT_PARAM = openapi.Parameter( + "attachments", + openapi.IN_FORM, + description="Повторяемые RTF-вложения; каждый файл проверяется по сигнатуре и SHA-256", + type=openapi.TYPE_FILE, + required=False, +) RESULT_LIST_PARAMS = [ PAGE_PARAM, PAGE_SIZE_PARAM, @@ -1539,6 +1555,11 @@ class SourceCardRefreshView(APIView): slug=slug, requested_by_id=request.user.id if request.user.is_authenticated else None, params=serializer.validated_data.get("params", {}), + mode=( + serializer.validated_data.get("params", {}).get("mode") + if slug == "fedresurs-company-profile" + else None + ), ) output = SourceCardRefreshResponseSerializer(payload).data tasks = output.get("tasks", []) @@ -2911,6 +2932,8 @@ class ParserRunView(APIView): serializer_class = ( SroParserRunRequestSerializer if canonical_source_key == "sro_membership_check" + else FedresursParserRunRequestSerializer + if canonical_source_key == "fedresurs_company_profile" else SourceCardRefreshRequestSerializer ) serializer = serializer_class(data=request.data) @@ -3096,13 +3119,74 @@ class ParserUploadView(APIView): permission_classes = [IsAuthenticated] parser_classes = [MultiPartParser, FormParser] - def post(self, request: Request, source_key: str): + def post(self, request: Request, source_key: str): # noqa: C901 descriptor = PARSER_SOURCES.get(source_key) if descriptor is None: return _source_not_found_response(source_key) if descriptor.admin_only and not request.user.is_staff: return Response(status=status.HTTP_403_FORBIDDEN) + if source_key == ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS: + from apps.parsers.integrated_structure_workbook import ( + IntegratedWorkbookError, + ) + from apps.parsers.integrated_structures import import_integrated_structures + + workbook = request.FILES.get("workbook") + if workbook is None or not workbook.name.lower().endswith(".xlsx"): + return api_error_response( + [ + { + "code": "workbook_required", + "field": "workbook", + "message": "Требуется XLSX workbook", + } + ], + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + if workbook.size > 20 * 1024 * 1024: + return api_error_response( + [{"code": "workbook_too_large", "message": "XLSX слишком велик"}], + status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + ) + job = BackgroundJobService.create_job( + task_id=str(uuid.uuid4()), + task_name=descriptor.task_name, + user_id=request.user.id, + meta={"source": descriptor.source, "source_key": source_key}, + ) + job.mark_started() + try: + artifact, result = import_integrated_structures( + workbook=workbook, + original_name=workbook.name, + attachments=request.FILES.getlist("attachments"), + uploaded_by_id=request.user.id, + ) + except IntegratedWorkbookError as exc: + job.fail(error=str(exc)) + return api_error_response( + [{"code": str(exc), "message": "Некорректный набор документов"}], + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + except Exception as exc: + job.fail(error=type(exc).__name__) + raise + job.complete(result={**result, "artifact_id": str(artifact.uid)}) + return api_response( + {**result, "artifact_id": str(artifact.uid), "task_id": job.task_id}, + status_code=status.HTTP_200_OK, + ) if not descriptor.supports_file_upload: + if source_key == ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE: + return api_error_response( + [ + { + "code": "upload_not_supported", + "message": "Федресурс обновляется только через публичный UI", + } + ], + status_code=status.HTTP_405_METHOD_NOT_ALLOWED, + ) return api_error_response( [ { diff --git a/src/organizations/fedresurs_payload.py b/src/organizations/fedresurs_payload.py new file mode 100644 index 0000000..f47d7e8 --- /dev/null +++ b/src/organizations/fedresurs_payload.py @@ -0,0 +1,90 @@ +"""Expose only verified public-UI fields; retain unparsed rendered sections as provenance.""" + +from __future__ import annotations + + +def _section(payload, name): + state = payload.get("sections", {}).get(name, "not_available") + if state == "loaded": + # The section is visible, but its nested document schema has not been + # verified by the adapter. Never claim its items were fully collected. + state = "schema_unverified" + return { + "state": state, + "found": None, + "loaded_at": payload.get("captured_at"), + "items": [], + "error_code": None, + } + + +def _unverified(payload, name): + section = _section(payload, name) + return { + "state": section["state"], + "found": None, + "endpoint": payload.get("source_url", ""), + "reason": payload.get("rendered_sections", {}).get(name, {}).get("text"), + } + + +def detail_payload(payload: dict, *, artifact=None) -> dict: + profile = dict(payload.get("profile") or {}) + profile.update( + fedresurs_guid=payload.get("fedresurs_guid"), + short_name=payload.get("short_name"), + kpp_largest_taxpayer=None, + registration_date_raw=None, + status={**profile.get("status", {}), "ended_at": None}, + okopf=None, + okved=None, + not_in_egrul=None, + roles=[], + unreliable_info={"address": None, "individual_executive_body": None}, + contacts=None, + trade_place=None, + ) + captured_at = payload.get("captured_at", "") + batch = str(artifact.load_batch) if artifact is not None else "" + return { + "profile": profile, + "executive_bodies": _section(payload, "executive_bodies"), + "publications": _section(payload, "publications"), + "bankrupt_publisher_publications": _section(payload, "bankruptcy"), + "licenses": _section(payload, "licenses"), + "net_worth": _section(payload, "net_worth"), + "reorganizations": { + "state": _section(payload, "reorganizations")["state"], + "loaded_at": captured_at, + "value": None, + "error_code": None, + }, + "encumbrances": _unverified(payload, "encumbrances"), + "sro": _unverified(payload, "sro"), + "sro_documents": _unverified(payload, "sro_documents"), + "sro_memberships": _unverified(payload, "sro_memberships"), + "bankruptcy": { + "state": _section(payload, "bankruptcy")["state"], + "intention_messages_count": 0, + "legal_cases_count": 0, + "without_legal_cases_count": 0, + "error_code": None, + }, + "biddings": _unverified(payload, "biddings"), + "pledged_subjects": _unverified(payload, "pledged_subjects"), + "account_receivables": _unverified(payload, "account_receivables"), + "russian_post_estates": _unverified(payload, "russian_post_estates"), + "lineage": { + "source_url": payload.get("source_url", ""), + "search_value": profile.get("inn", ""), + "searched_at": captured_at, + "captured_at": captured_at, + "adapter_version": "public-ui/v1", + "raw_batch_id": batch, + "content_hash": payload.get("content_hash", ""), + }, + "rendered_sections": payload.get("rendered_sections", {}), + "rendered_page": payload.get("rendered_page", {}), + "related_pages": payload.get("related_pages", {}), + "message_details": payload.get("message_details", []), + } diff --git a/src/organizations/migrations/0014_auto_20260924_1056.py b/src/organizations/migrations/0014_auto_20260924_1056.py new file mode 100644 index 0000000..1a8aee4 --- /dev/null +++ b/src/organizations/migrations/0014_auto_20260924_1056.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.25 on 2026-09-24 10:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('organizations', '0013_auto_20260914_1406'), + ] + + operations = [ + migrations.CreateModel( + name='IntegratedStructureExtension', + fields=[ + ('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')), + ], + options={ + 'verbose_name': 'документы интегрированной структуры', + 'verbose_name_plural': 'документы интегрированных структур', + 'db_table': 'organizations_integrated_structure_extension', + }, + bases=('organizations.organizationsourceextension',), + ), + migrations.AlterField( + model_name='organizationsourceextension', + name='source_group', + field=models.CharField(choices=[('financial_indicators', 'Финансово-экономические показатели'), ('government_procurements', 'Государственные закупки'), ('industrial_production', 'Производители и продукция России'), ('planned_inspections', 'Плановые проверки'), ('bankruptcy', 'Сведения о процедурах банкротства'), ('defense_suppliers', 'Недобросовестные поставщики ГОЗ'), ('arbitration', 'Арбитражные дела'), ('security_registries', 'Реестры по информационной безопасности'), ('vacancies', 'Вакансии'), ('electronic_document_exchange', 'Электронный документооборот'), ('media_mentions', 'Упоминания в СМИ'), ('sanctions', 'Санкции'), ('government_support', 'Государственная поддержка'), ('sro_membership', 'Членство в СРО'), ('budget_process_registry', 'Реестр участников бюджетного процесса'), ('integrated_structure_documents', 'Документы интегрированных структур')], db_index=True, max_length=64, verbose_name='группа источников'), + ), + ] diff --git a/src/organizations/migrations/0015_auto_20260924_1114.py b/src/organizations/migrations/0015_auto_20260924_1114.py new file mode 100644 index 0000000..89ab1c7 --- /dev/null +++ b/src/organizations/migrations/0015_auto_20260924_1114.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.25 on 2026-09-24 11:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('organizations', '0014_auto_20260924_1056'), + ] + + operations = [ + migrations.CreateModel( + name='FedresursCompanyProfileExtension', + fields=[ + ('organizationsourceextension_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='organizations.organizationsourceextension')), + ], + options={ + 'verbose_name': 'карточка компании Федресурса', + 'verbose_name_plural': 'карточки компаний Федресурса', + 'db_table': 'organizations_fedresurs_company_profile_extension', + }, + bases=('organizations.organizationsourceextension',), + ), + migrations.AlterField( + model_name='organizationsourceextension', + name='source_group', + field=models.CharField(choices=[('financial_indicators', 'Финансово-экономические показатели'), ('government_procurements', 'Государственные закупки'), ('industrial_production', 'Производители и продукция России'), ('planned_inspections', 'Плановые проверки'), ('bankruptcy', 'Сведения о процедурах банкротства'), ('defense_suppliers', 'Недобросовестные поставщики ГОЗ'), ('arbitration', 'Арбитражные дела'), ('security_registries', 'Реестры по информационной безопасности'), ('vacancies', 'Вакансии'), ('electronic_document_exchange', 'Электронный документооборот'), ('media_mentions', 'Упоминания в СМИ'), ('sanctions', 'Санкции'), ('government_support', 'Государственная поддержка'), ('sro_membership', 'Членство в СРО'), ('budget_process_registry', 'Реестр участников бюджетного процесса'), ('integrated_structure_documents', 'Документы интегрированных структур'), ('fedresurs_company_profile', 'Карточки компаний Федресурса')], db_index=True, max_length=64, verbose_name='группа источников'), + ), + ] diff --git a/src/organizations/models.py b/src/organizations/models.py index 408b9a1..ad26c1b 100644 --- a/src/organizations/models.py +++ b/src/organizations/models.py @@ -44,6 +44,14 @@ class SourceGroup(models.TextChoices): "budget_process_registry", _("Реестр участников бюджетного процесса"), ) + INTEGRATED_STRUCTURE_DOCUMENTS = ( + "integrated_structure_documents", + _("Документы интегрированных структур"), + ) + FEDRESURS_COMPANY_PROFILE = ( + "fedresurs_company_profile", + _("Карточки компаний Федресурса"), + ) class SourceExtensionStatus(models.TextChoices): @@ -742,6 +750,28 @@ class SroMembershipExtension(OrganizationSourceExtension): verbose_name_plural = _("членства в СРО") +class IntegratedStructureExtension(OrganizationSourceExtension): + """One canonical organization with imported integrated-structure documents.""" + + source_group_value = SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS + + class Meta: + db_table = "organizations_integrated_structure_extension" + verbose_name = _("документы интегрированной структуры") + verbose_name_plural = _("документы интегрированных структур") + + +class FedresursCompanyProfileExtension(OrganizationSourceExtension): + """Public Fedresurs company profile matched to a canonical organization.""" + + source_group_value = SourceGroup.FEDRESURS_COMPANY_PROFILE + + class Meta: + db_table = "organizations_fedresurs_company_profile_extension" + verbose_name = _("карточка компании Федресурса") + verbose_name_plural = _("карточки компаний Федресурса") + + class OrganizationSourceRecord(models.Model): """Subordinate source record stored under a source extension.""" diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index c094e8a..a306886 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -320,8 +320,70 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): @swagger_serializer_method( serializer_or_field=OrganizationSourceRecordDetailPayloadSerializer, ) - def get_payload(self, obj) -> dict | list | str | int | float | bool | None: + def get_payload(self, obj) -> dict | list | str | int | float | bool | None: # noqa: C901 payload = obj.payload + if obj.source == "fedresurs_company_profile": + from organizations.fedresurs_payload import detail_payload + + return detail_payload(payload) + if obj.source == "integrated_structures": + response_payload = dict(payload) if isinstance(payload, dict) else {} + view = self.context.get("view") + if getattr(view, "action", None) == "retrieve": + events = [] + for event in obj.integrated_events.prefetch_related( + "document_links__document__attachment" + ).all(): + documents = [] + for link in event.document_links.all(): + document = link.document + attachment = document.attachment + documents.append( + { + "id": document.external_id, + "name": document.name, + "type": document.document_type, + "number": document.number, + "date": document.document_date.isoformat() + if document.document_date + else "", + "dsp": document.dsp, + "have_text": document.have_text, + "consultant_web_url": document.consultant_web_url + or None, + "attachment": { + "available": attachment is not None, + "download_allowed": attachment is not None, + "filename": document.attachment_filename + if attachment + else None, + "mime_type": "application/rtf" + if attachment + else None, + "sha256": attachment.sha256 if attachment else None, + "size": attachment.size_bytes + if attachment + else None, + }, + } + ) + documents.sort( + key=lambda item: (item["date"], item["id"]), reverse=True + ) + events.append( + { + "id": event.external_id, + "date": event.event_date.isoformat() + if event.event_date + else "", + "note": event.note or None, + "documents_count": len(documents), + "documents": documents, + } + ) + events.sort(key=lambda item: (item["date"], item["id"]), reverse=True) + response_payload["events"] = events + return response_payload if obj.source in ("budget_ubpandnubp", "fns_sme_support_recipients"): context = self.context.get("registry_payload_context") if context is None: @@ -410,6 +472,24 @@ class OrganizationSourceRecordListSerializer(OrganizationSourceRecordSerializer) serializer_or_field=OrganizationSourceRecordPayloadSerializer, ) def get_payload(self, obj) -> dict | list | str | int | float | bool | None: + if obj.source == "fedresurs_company_profile": + return { + key: obj.payload.get(key) + for key in ( + "fedresurs_guid", + "short_name", + "kpp", + "status", + "executive_body_name", + "registration_date", + "publication_count", + "reorganization_count", + "has_bankruptcy_information", + "captured_at", + "source_url", + "sections", + ) + } if obj.source in ("budget_ubpandnubp", "fns_sme_support_recipients"): return serialize_registry_payload( obj, diff --git a/src/organizations/source_groups.py b/src/organizations/source_groups.py index 4b4977b..f1f8131 100644 --- a/src/organizations/source_groups.py +++ b/src/organizations/source_groups.py @@ -12,10 +12,12 @@ from organizations.models import ( BudgetProcessRegistryExtension, DefenseSupplierExtension, ElectronicDocumentExchangeExtension, + FedresursCompanyProfileExtension, FinancialIndicatorsExtension, GovernmentProcurementExtension, GovernmentSupportExtension, IndustrialProductionExtension, + IntegratedStructureExtension, MediaMentionExtension, OrganizationSourceExtension, PlannedInspectionExtension, @@ -39,6 +41,20 @@ class SourceGroupDescriptor: SOURCE_GROUP_DESCRIPTORS: dict[str, SourceGroupDescriptor] = { + ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE: SourceGroupDescriptor( + source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, + source_group=SourceGroup.FEDRESURS_COMPANY_PROFILE, + record_type="company_profile", + title="Карточка компании Федресурса", + extension_model=FedresursCompanyProfileExtension, + ), + "integrated_structures": SourceGroupDescriptor( + source="integrated_structures", + source_group=SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS, + record_type="integrated_structure", + title="Документы интегрированных структур", + extension_model=IntegratedStructureExtension, + ), ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK: SourceGroupDescriptor( source=ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK, source_group=SourceGroup.SRO_MEMBERSHIP, diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index 8aac96d..3615c92 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -2,6 +2,7 @@ from __future__ import annotations +import base64 import csv import json import math @@ -22,7 +23,11 @@ from tempfile import NamedTemporaryFile from typing import Any, BinaryIO, cast from uuid import uuid4 -from apps.parsers.models import ParserSourceArtifact +from apps.parsers.models import ( + IntegratedStructureEvent, + IntegratedStructureEventDocument, + ParserSourceArtifact, +) from django.conf import settings from django.core.cache import cache from django.db import connection, transaction @@ -48,6 +53,8 @@ ALL_HISTORY_SOURCE_GROUPS = { SourceGroup.BUDGET_PROCESS_REGISTRY.value, SourceGroup.MEDIA_MENTIONS.value, SourceGroup.SANCTIONS.value, + SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS.value, + SourceGroup.FEDRESURS_COMPANY_PROFILE.value, } EXPORT_MANIFEST_VERSION = 2 CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json" @@ -56,6 +63,7 @@ GENERATION_DIRECTORY_NAME = "generations" SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE = 1000 SOURCE_RECORD_EXPORT_ZIP_CHUNK_SIZE = 1024 * 1024 EXCEL_MAX_DATA_ROWS_PER_SHEET = 1_048_575 +EXCEL_SAFE_CELL_CHARACTERS = 30_000 DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000 DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60 SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket" @@ -74,11 +82,15 @@ SNAPSHOT_EXPORT_SOURCES = { "budget_process_registry": "budget_ubpandnubp", "government_support": "fns_sme_support_recipients", "sro_membership": "sro_membership_check", + "integrated_structure_documents": "integrated_structures", + "fedresurs_company_profile": "fedresurs_company_profile", } GENERATION_ID_PATTERN = re.compile(r"[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}") SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = { SourceGroup.SRO_MEMBERSHIP.value: "sro-membership-check", + SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS.value: "integrated-structure-documents", + SourceGroup.FEDRESURS_COMPANY_PROFILE.value: "fedresurs-company-profile", SourceGroup.GOVERNMENT_SUPPORT.value: "sme-support-recipients-registry", SourceGroup.BUDGET_PROCESS_REGISTRY.value: "budget-process-registry", SourceGroup.FINANCIAL_INDICATORS.value: "financial-indicators", @@ -276,6 +288,8 @@ def build_source_record_export_artifacts( source_record_counts: dict[str, int] = {} spooled_groups = [] + integrated_event_spool = staging_directory / ".integrated.events.jsonl" + integrated_link_spool = staging_directory / ".integrated.links.jsonl" with _source_export_read_snapshot(): source_snapshots = _published_snapshot_ids() for source_group in SOURCE_GROUP_EXPORT_FILE_STEMS: @@ -289,6 +303,7 @@ def build_source_record_export_artifacts( spooled_groups.append( (source_group, row_spool_path, headers, records_count) ) + _spool_integrated_relations(integrated_event_spool, integrated_link_spool) if _published_snapshot_ids() != source_snapshots: raise SourceRecordExportArtifactsUnavailable( "Published snapshot changed during export preparation." @@ -301,18 +316,31 @@ def build_source_record_export_artifacts( source_group=source_group, file_format=file_format, ) - artifact_paths = _render_source_group_artifact( - row_spool_path=row_spool_path, - output_path=staging_directory / file_name, - headers=headers, - file_format=file_format, - records_count=records_count, - display_headers=( - SANCTIONS_XLSX_HEADERS - if source_group == SourceGroup.SANCTIONS - else None - ), - ) + if ( + source_group == SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS + and file_format == EXPORT_FORMAT_XLSX + ): + output_path = staging_directory / file_name + _render_integrated_xlsx( + row_spool_path, + integrated_event_spool, + integrated_link_spool, + output_path, + ) + artifact_paths = (output_path,) + else: + artifact_paths = _render_source_group_artifact( + row_spool_path=row_spool_path, + output_path=staging_directory / file_name, + headers=headers, + file_format=file_format, + records_count=records_count, + display_headers=( + SANCTIONS_XLSX_HEADERS + if source_group == SourceGroup.SANCTIONS + else None + ), + ) parts_count = len(artifact_paths) artifacts.extend( SourceRecordExportArtifact( @@ -332,6 +360,8 @@ def build_source_record_export_artifacts( ) finally: row_spool_path.unlink(missing_ok=True) + integrated_event_spool.unlink(missing_ok=True) + integrated_link_spool.unlink(missing_ok=True) generation = SourceRecordExportGeneration( generation_id=generation_id, @@ -851,6 +881,146 @@ def _iter_spooled_rows(row_spool_path: Path) -> Iterator[dict[str, Any]]: yield json.loads(serialized_row) +def _spool_integrated_relations(event_path: Path, link_path: Path) -> None: + """Freeze relational rows under the same read snapshot as public records.""" + with event_path.open("w", encoding="utf-8") as output: + events = ( + IntegratedStructureEvent.objects.filter( + record__source="integrated_structures" + ) + .select_related("record__extension__organization") + .order_by("record_id", "external_id") + ) + for event in events.iterator(chunk_size=1000): + output.write( + json.dumps( + { + "record_uid": str(event.record.uid), + "okpo": event.record.extension.organization.okpo, + "event_id": event.external_id, + "event_date": event.event_date.isoformat() + if event.event_date + else "", + "note": event.note, + }, + ensure_ascii=False, + ) + + "\n" + ) + with link_path.open("w", encoding="utf-8") as output: + links = ( + IntegratedStructureEventDocument.objects.filter( + event__record__source="integrated_structures" + ) + .select_related( + "event__record__extension__organization", "document__attachment" + ) + .order_by("event__record_id", "event__external_id", "document__external_id") + ) + for link in links.iterator(chunk_size=1000): + document = link.document + output.write( + json.dumps( + { + "record_uid": str(link.event.record.uid), + "okpo": link.event.record.extension.organization.okpo, + "event_id": link.event.external_id, + "document_id": document.external_id, + "document_name": document.name, + "document_type": document.document_type, + "document_number": document.number, + "document_date": document.document_date.isoformat() + if document.document_date + else "", + "dsp": document.dsp, + "have_text": document.have_text, + "consultant_web_url": document.consultant_web_url, + "attachment_available": document.attachment_id is not None, + "attachment_filename": document.attachment_filename, + "attachment_sha256": document.attachment_id or "", + }, + ensure_ascii=False, + ) + + "\n" + ) + + +def _render_integrated_xlsx( + structure_path: Path, event_path: Path, link_path: Path, output_path: Path +) -> None: + """Export one relational workbook; never include attachment file bytes.""" + workbook = Workbook(write_only=True) + structures = workbook.create_sheet(title="structures") + structure_fields = ( + "uid", + "ОКПО", + "Наименование", + "ИНН", + "ОГРН", + "Событий", + "Связей с документами", + "Уникальных документов", + "Первое событие", + "Последнее событие", + ) + structures.append(structure_fields) + for row in _iter_spooled_rows(structure_path): + structures.append( + [ + _serialize_spreadsheet_value(value) + for value in ( + row.get("uid"), + row.get("ОКПО"), + row.get("Наименование"), + row.get("ИНН"), + row.get("ОГРН"), + row.get("payload.structure.events_count"), + row.get("payload.structure.document_links_count"), + row.get("payload.structure.unique_documents_count"), + row.get("payload.structure.first_event_date"), + row.get("payload.structure.latest_event_date"), + ) + ] + ) + for title, source_path, fields in ( + ( + "events", + event_path, + ("record_uid", "okpo", "event_id", "event_date", "note"), + ), + ( + "event_documents", + link_path, + ( + "record_uid", + "okpo", + "event_id", + "document_id", + "document_name", + "document_type", + "document_number", + "document_date", + "dsp", + "have_text", + "consultant_web_url", + "attachment_available", + "attachment_filename", + "attachment_sha256", + ), + ), + ): + sheet = workbook.create_sheet(title=title) + sheet.append(fields) + with source_path.open("r", encoding="utf-8") as source: + for line in source: + row = json.loads(line) + sheet.append( + [_serialize_spreadsheet_value(row.get(field)) for field in fields] + ) + workbook.save(output_path) + workbook.close() + + def _render_csv_file( *, row_spool_path: Path, @@ -899,6 +1069,9 @@ def _render_xlsx_files( part_number = 1 rows_in_file = 0 workbook, worksheet = _new_export_workbook(display_headers or headers) + overflow_sheet = None + overflow_rows = 0 + overflow_part = 0 for row in _iter_spooled_rows(row_spool_path): if rows_in_file >= rows_per_file: @@ -907,9 +1080,48 @@ def _render_xlsx_files( part_number += 1 rows_in_file = 0 workbook, worksheet = _new_export_workbook(display_headers or headers) - worksheet.append( - [_serialize_spreadsheet_value(row.get(key)) for key in headers] - ) + overflow_sheet = None + overflow_rows = 0 + overflow_part = 0 + cells = [] + for key in headers: + value = _serialize_flat_value(row.get(key)) + if not isinstance(value, str) or ( + len(value.encode("utf-16-le")) // 2 <= EXCEL_SAFE_CELL_CHARACTERS + ): + cells.append(_serialize_spreadsheet_value(row.get(key))) + continue + if overflow_sheet is None: + overflow_part += 1 + overflow_sheet = workbook.create_sheet( + title=f"overflow_values_{overflow_part}" + ) + overflow_sheet.append( + ["record_uid", "field_name", "part", "encoding", "chunk"] + ) + overflow_rows = 0 + encoded = base64.b64encode(value.encode("utf-8")).decode("ascii") + for chunk_number, offset in enumerate( + range(0, len(encoded), EXCEL_SAFE_CELL_CHARACTERS), 1 + ): + if overflow_rows >= EXCEL_MAX_DATA_ROWS_PER_SHEET: + overflow_part += 1 + overflow_sheet = workbook.create_sheet( + title=f"overflow_values_{overflow_part}" + ) + overflow_sheet.append( + ["record_uid", "field_name", "part", "encoding", "chunk"] + ) + overflow_rows = 0 + overflow_sheet.append( + [ + str(row.get("uid", "")), key, chunk_number, + "utf-8+base64", "b64:" + encoded[offset:offset + EXCEL_SAFE_CELL_CHARACTERS], + ] + ) + overflow_rows += 1 + cells.append(f"См. overflow_values: {key}") + worksheet.append(cells) rows_in_file += 1 workbook.save(part_paths[part_number - 1]) diff --git a/src/organizations/source_record_query.py b/src/organizations/source_record_query.py index f246390..dfc8d09 100644 --- a/src/organizations/source_record_query.py +++ b/src/organizations/source_record_query.py @@ -9,20 +9,34 @@ from decimal import Decimal, InvalidOperation from uuid import UUID from apps.core.pagination import StandardPagination -from django.db.models import BooleanField, Case, F, Func, Q, TextField, Value, When +from django.db.models import ( + BooleanField, + Case, + F, + Func, + IntegerField, + Q, + TextField, + Value, + When, +) from django.db.models.fields.json import KeyTextTransform -from django.db.models.functions import Coalesce, NullIf +from django.db.models.functions import Cast, Coalesce, NullIf from drf_yasg import openapi REGISTRY_GROUPS = ( "budget_process_registry", "government_support", "sro_membership", + "integrated_structure_documents", + "fedresurs_company_profile", ) REGISTRY_SOURCE_GROUPS = { "budget_ubpandnubp": "budget_process_registry", "fns_sme_support_recipients": "government_support", "sro_membership_check": "sro_membership", + "integrated_structures": "integrated_structure_documents", + "fedresurs_company_profile": "fedresurs_company_profile", } REGISTRY_ORDERING_FIELDS = ( @@ -46,6 +60,12 @@ REGISTRY_ORDERING_FIELDS = ( "payload__region", "payload__sro_name", "payload__admission_date", + "payload__status__name", + "payload__executive_body_name", + "payload__registration_date", + "payload__publication_count", + "payload__reorganization_count", + "payload__captured_at", ) EXACT_FILTERS = { "organization_type": "payload__classification__organization_type__code", @@ -63,13 +83,23 @@ BOOLEAN_FILTERS = { "is_branch": "payload__is_separate_division", "has_procurement_permission": "payload__summary__has_procurement_permission", "has_violation": "payload__has_violation", + "has_bankruptcy_information": "payload__has_bankruptcy_information", } +FEDRESURS_COUNT_FILTERS = ("has_publications", "has_reorganizations") +FEDRESURS_SECTION_STATES = ( + "loaded", + "empty", + "not_available", + "blocked", + "schema_unverified", +) DATE_RANGES = { "date": "canonical_record_date", "registry_entry": "payload__registry_entry_date", "support_until": "payload__support_until", "termination": "payload__termination_date", "admission_date": "payload__admission_date", + "registration_date": "payload__registration_date", } ALIASES = {"support_form": "support_form_code", "support_kind": "support_kind_code"} UNIT_CHOICES = ("RUB", "square_meter", "hour", "percent", "unit") @@ -89,12 +119,15 @@ COMMON_PARAMS = { FILTER_PARAMS = ( set(EXACT_FILTERS) | set(BOOLEAN_FILTERS) + | set(FEDRESURS_COUNT_FILTERS) | set(ALIASES) | { "region_code", "support_unit", "amount_from", "amount_to", + "company_status", + "section_state", } | {f"{prefix}_{suffix}" for prefix in DATE_RANGES for suffix in ("from", "to")} ) @@ -186,7 +219,7 @@ def _validate_aliases_and_booleans(values, error): if canonical in values and values[alias] != values[canonical]: error(alias, "conflicting_filters") values.setdefault(canonical, values[alias]) - for name in BOOLEAN_FILTERS: + for name in (*BOOLEAN_FILTERS, *FEDRESURS_COUNT_FILTERS): if name in values: if values[name] not in ("true", "false", "1", "0"): error(name, "invalid_boolean") @@ -215,6 +248,17 @@ def _validate_choices(values, error): error("status") if "support_unit" in values and values["support_unit"] not in UNIT_CHOICES: error("support_unit") + if "company_status" in values and values["company_status"] not in ( + "active", + "liquidating", + "terminated", + ): + error("company_status") + if ( + "section_state" in values + and values["section_state"] not in FEDRESURS_SECTION_STATES + ): + error("section_state") def _validate_dates(values, error): @@ -259,10 +303,41 @@ def _validate_amounts(values, error): error("amount_from", "invalid_amount_range") -def apply_registry_filters(queryset, values): +def apply_registry_filters(queryset, values): # noqa: C901 for name, lookup in {**EXACT_FILTERS, **BOOLEAN_FILTERS}.items(): if name in values: queryset = queryset.filter(**{lookup: values[name]}) + if "company_status" in values: + queryset = queryset.filter(payload__status__code=values["company_status"]) + for filter_name, payload_key in ( + ("has_publications", "publication_count"), + ("has_reorganizations", "reorganization_count"), + ): + if filter_name in values: + lookup = "gt" if values[filter_name] else "lte" + queryset = queryset.filter(**{f"payload__{payload_key}__{lookup}": 0}) + if "section_state" in values: + section_query = Q() + for section in ( + "profile", + "executive_bodies", + "publications", + "encumbrances", + "sro", + "sro_memberships", + "licenses", + "net_worth", + "reorganizations", + "bankruptcy", + "biddings", + "pledged_subjects", + "account_receivables", + "russian_post_estates", + ): + section_query |= Q( + **{f"payload__sections__{section}": values["section_state"]} + ) + queryset = queryset.filter(section_query) if "region_code" in values: region = values["region_code"] queryset = queryset.filter( @@ -330,6 +405,16 @@ def source_specific_search(term): def ordering_expression(field): + if field in ("events_count", "documents_count"): + key = "events_count" if field == "events_count" else "document_links_count" + return Cast( + KeyTextTransform(key, KeyTextTransform("structure", F("payload"))), + IntegerField(), + ) + if field == "first_event_date": + return KeyTextTransform( + "first_event_date", KeyTextTransform("structure", F("payload")) + ) if field.startswith("organization__"): field = "extension__" + field if field.startswith("extension__organization__"): @@ -384,7 +469,7 @@ def registry_query_parameters(): parameters = [] for name in sorted(FILTER_PARAMS - {"date_from", "date_to"}): kwargs = {"type": openapi.TYPE_STRING} - if name in BOOLEAN_FILTERS: + if name in BOOLEAN_FILTERS or name in FEDRESURS_COUNT_FILTERS: kwargs["type"] = openapi.TYPE_BOOLEAN elif any( name == f"{prefix}_{suffix}" diff --git a/src/organizations/test_companies.py b/src/organizations/test_companies.py index 78ab90e..fb1b615 100644 --- a/src/organizations/test_companies.py +++ b/src/organizations/test_companies.py @@ -45,6 +45,8 @@ CANONICAL_ONLY_TEST_SOURCES = { } FILE_UPLOAD_ONLY_TEST_SOURCES = { ParserLoadLog.Source.ROPK_SANCTIONS, + ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS, + ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, } TEST_BALANCE_LINE_NAMES = { diff --git a/src/organizations/views.py b/src/organizations/views.py index 6572c1c..4fb6b06 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -15,7 +15,7 @@ from django.core.cache import cache from django.db.models import Case, CharField, F, Q, Value, When from django.db.models.fields.json import KeyTextTransform from django.db.models.functions import Coalesce, NullIf -from django.http import Http404, StreamingHttpResponse +from django.http import FileResponse, Http404, StreamingHttpResponse from django_filters import rest_framework as filters from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -144,6 +144,9 @@ SOURCE_RECORD_ORDERING_FIELDS = ( "payload__ukraine", "payload__sentiment", "payload__news_source", + "events_count", + "documents_count", + "first_event_date", *REGISTRY_ORDERING_FIELDS, ) SOURCE_RECORD_ORDERING_VALUES = [ @@ -587,6 +590,8 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): return OrganizationSourceRecordSerializer def get_permissions(self): + if self.action == "integrated_attachment": + return [IsAuthenticated()] if self.action in {"export", "export_ticket"}: return [IsAdminUser()] if self.action == "export_download": @@ -778,6 +783,57 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): "source_record_not_found", "Запись не найдена", 404 ) + @swagger_auto_schema( + tags=[ORGANIZATIONS_TAG], + operation_id="api_v2_organization_source_records_documents_attachment_read", + operation_summary="Скачать RTF интегрированной структуры", + responses={ + 200: openapi.Response( + description="RTF-вложение", + schema=openapi.Schema(type=openapi.TYPE_FILE), + ), + 401: SourceRecordErrorResponseSerializer, + 404: SourceRecordErrorResponseSerializer, + 409: SourceRecordErrorResponseSerializer, + }, + ) + @action( + detail=True, + methods=["get"], + url_path=r"documents/(?P[0-9]+)/attachment", + ) + def integrated_attachment(self, request, uid=None, document_id=None): + from apps.parsers.models import IntegratedStructureEventDocument + + record = self.get_object() + if record.source != "integrated_structures": + return self._record_error( + "source_record_not_found", "Запись не найдена", 404 + ) + link = ( + IntegratedStructureEventDocument.objects.filter( + event__record=record, document__external_id=document_id + ) + .select_related("document__attachment") + .first() + ) + if link is None: + return self._record_error("document_not_found", "Документ не найден", 404) + attachment = link.document.attachment + if attachment is None: + return self._record_error( + "attachment_unavailable", "RTF-файл не загружен", 409 + ) + response = FileResponse( + attachment.file.open("rb"), + content_type="application/rtf", + as_attachment=True, + filename=link.document.attachment_filename, + ) + response["Cache-Control"] = "private, no-store" + response["X-Content-Type-Options"] = "nosniff" + return response + @staticmethod def _record_error(code: str, message: str, http_status: int) -> Response: return Response( diff --git a/src/settings/base.py b/src/settings/base.py index 24bf6a5..51b1bf6 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -334,6 +334,18 @@ PARSER_PRIVATE_ARTIFACT_ROOT = Path( "PARSER_PRIVATE_ARTIFACT_ROOT", str(PROJECT_ROOT / "private-parser-artifacts") ) ) +FEDRESURS_COMPANY_PROFILE_ENABLED = ( + os.getenv("FEDRESURS_COMPANY_PROFILE_ENABLED", "false").lower() == "true" +) +FEDRESURS_BROWSER_BINARY = os.getenv( + "FEDRESURS_BROWSER_BINARY", "/opt/chrome/chrome-linux64/chrome" +) +FEDRESURS_BROWSER_DRIVER = os.getenv( + "FEDRESURS_BROWSER_DRIVER", "/opt/chrome/chromedriver-linux64/chromedriver" +) +FEDRESURS_PAGE_TIMEOUT_SECONDS = int(os.getenv("FEDRESURS_PAGE_TIMEOUT_SECONDS", "45")) +FEDRESURS_MIN_INTERVAL_SECONDS = float(os.getenv("FEDRESURS_MIN_INTERVAL_SECONDS", "2")) +FEDRESURS_MAX_RETRIES = int(os.getenv("FEDRESURS_MAX_RETRIES", "2")) # Default primary key field type DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" diff --git a/tests/apps/organizations/test_source_export_snapshot_contract.py b/tests/apps/organizations/test_source_export_snapshot_contract.py index 14e05ed..725c5a5 100644 --- a/tests/apps/organizations/test_source_export_snapshot_contract.py +++ b/tests/apps/organizations/test_source_export_snapshot_contract.py @@ -210,7 +210,7 @@ def test_sro_export_keeps_all_years_and_identifier_strings(export_client, file_f record_date="2001-01-01", ) generation = exports.build_source_record_export_artifacts() - assert generation.artifacts_count == 43 + assert generation.artifacts_count == 49 response = issue(export_client, "sro_membership", file_format) assert response.status_code == 201 package = exports.consume_source_record_export_download_ticket( diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index d5094bc..518dd15 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -1,5 +1,6 @@ """Tests for organization source-record ZIP export.""" +import base64 import csv import json import os @@ -28,6 +29,7 @@ from organizations.models import ( from organizations.source_record_export import ( SourceRecordExportArtifactsUnavailable, _render_source_group_artifact, + _render_xlsx_files, _source_group_queryset, _spool_source_group_rows, build_source_record_export_artifacts, @@ -71,6 +73,32 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): return b"".join(response.streaming_content) return response.content + def test_xlsx_export_preserves_large_rendered_fedresurs_field_in_chunks(self): + value = "Атрибут 😀" * 6_000 + with TemporaryDirectory() as directory: + spool = Path(directory) / "rows.jsonl" + spool.write_text( + json.dumps( + {"uid": "record-1", "payload.rendered_page.text": value}, + ensure_ascii=False, + ) + "\n", + encoding="utf-8", + ) + paths = _render_xlsx_files( + row_spool_path=spool, + output_path=Path(directory) / "fedresurs.xlsx", + headers=["uid", "payload.rendered_page.text"], + records_count=1, + ) + workbook = load_workbook(paths[0], read_only=True) + self.assertEqual(workbook["data"]["B2"].value, "См. overflow_values: payload.rendered_page.text") + chunks = [ + row[4][4:] + for row in list(workbook["overflow_values_1"].values)[1:] + ] + self.assertEqual(base64.b64decode("".join(chunks)).decode("utf-8"), value) + workbook.close() + def test_export_returns_service_unavailable_before_first_nightly_generation(self): self.client.force_authenticate(UserFactory.create_superuser()) @@ -93,9 +121,9 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): call_command("build_source_record_exports", stdout=command_output) generation = load_current_source_record_export_generation() - self.assertEqual(generation.artifacts_count, 43) + self.assertEqual(generation.artifacts_count, 49) self.assertEqual(generation.export_year, timezone.localdate().year) - self.assertIn('"artifacts_count": 43', command_output.getvalue()) + self.assertIn('"artifacts_count": 49', command_output.getvalue()) def test_sanctions_xlsx_export_uses_source_contract_columns_and_boolean_flags(self): organization = Organization.objects.create( @@ -466,7 +494,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): self.assertEqual( response["X-Source-Export-Generated-At"], generation.generated_at ) - self.assertEqual(generation.artifacts_count, 43) + self.assertEqual(generation.artifacts_count, 49) self.assertIn( 'filename="planned-inspections__financial-indicators_', response["Content-Disposition"], @@ -661,8 +689,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): key=lambda item: item.file_name, ) - self.assertEqual(generation.artifacts_count, 43) - self.assertEqual(generation.files_count, 44) + self.assertEqual(generation.artifacts_count, 49) + self.assertEqual(generation.files_count, 50) self.assertEqual( [item.file_name for item in artifacts], [ @@ -860,7 +888,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): first_generation = build_source_record_export_artifacts() current_generation = load_current_source_record_export_generation() - self.assertEqual(first_generation.artifacts_count, 43) + self.assertEqual(first_generation.artifacts_count, 49) self.assertEqual( current_generation.generation_id, first_generation.generation_id ) diff --git a/tests/apps/organizations/test_tasks.py b/tests/apps/organizations/test_tasks.py index 3c60a89..58e0545 100644 --- a/tests/apps/organizations/test_tasks.py +++ b/tests/apps/organizations/test_tasks.py @@ -147,7 +147,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase): result = refresh_source_record_export_artifacts() self.assertEqual(result["status"], "success") - self.assertEqual(result["artifacts_count"], 43) + self.assertEqual(result["artifacts_count"], 49) self.assertEqual(result["export_year"], timezone.localdate().year) self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY)) diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py index 52d1b32..d605f5d 100644 --- a/tests/apps/organizations/test_test_companies_commands.py +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -66,11 +66,20 @@ class TestCompaniesCommandsTest(TestCase): 10, ) - expected_groups = {choice.value for choice in SourceGroup} + expected_groups = { + choice.value + for choice in SourceGroup + if choice is not SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS + } expected_sources = { choice.value for choice in ParserLoadLog.Source - if choice is not ParserLoadLog.Source.ROPK_SANCTIONS + if choice + not in { + ParserLoadLog.Source.ROPK_SANCTIONS, + ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS, + ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, + } } for company in companies: self.assertEqual( @@ -264,7 +273,7 @@ class TestCompaniesCommandsTest(TestCase): ) self.assertEqual( OrganizationSourceRecord.objects.count(), - 20 * (len(ParserLoadLog.Source) - 1), + 20 * (len(ParserLoadLog.Source) - 3), ) def test_create_replaces_stale_source_record_for_same_legacy_row(self): diff --git a/tests/apps/parsers/test_fedresurs_company_profile.py b/tests/apps/parsers/test_fedresurs_company_profile.py new file mode 100644 index 0000000..a918562 --- /dev/null +++ b/tests/apps/parsers/test_fedresurs_company_profile.py @@ -0,0 +1,146 @@ +from apps.parsers.fedresurs_browser import _related_company_urls, _wait_public_element +from apps.parsers.fedresurs_company_profile import ( + FedresursUpstreamError, + parse_profile_html, + parse_related_page_html, + parse_search_html, +) +from django.test import SimpleTestCase + +SEARCH_HTML = """ + +
АО Тест
ИНН
6150061451
+
ОГРН
1096100003625
+
+""" +PROFILE_HTML = """ +
АО Тест
Действующее
+
Общая информация
+
ИНН
6150061451
+
ОГРН
1096100003625
+
КПП
772601001
+
Полное наименование
АКЦИОНЕРНОЕ ОБЩЕСТВО ТЕСТ
+
Дата регистрации
13.11.2009
+
+
Публикации
Нет данных
+
Обременения
Нет данных
+
Единоличный исполнительный орган
Нет данных
+
Сведения о СРО
Нет данных
+
Членство в СРО
Нет данных
+
Лицензии
Нет данных
+
Стоимость чистых активов
Нет данных
+
Сведения о реорганизации
Нет данных
+
Сведения о банкротстве
Нет данных
+
Торги
Нет данных
+""" + + +class FedresursDomTest(SimpleTestCase): + def test_exact_search_match(self): + result = parse_search_html(SEARCH_HTML, inn="6150061451", ogrn="1096100003625") + self.assertEqual(result, "match") + + def test_search_never_accepts_partial_identifier(self): + result = parse_search_html(SEARCH_HTML, inn="615006145", ogrn="1096100003625") + self.assertEqual(result, "no_match") + + def test_duplicate_exact_search_is_ambiguous(self): + result = parse_search_html(SEARCH_HTML + SEARCH_HTML, inn="6150061451", ogrn="1096100003625") + self.assertEqual(result, "ambiguous") + + def test_profile_requires_complete_identity_and_sections(self): + profile = parse_profile_html( + PROFILE_HTML, + url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", + inn="6150061451", ogrn="1096100003625", + ) + self.assertEqual(profile["fedresurs_guid"], "e1661e0e-d217-4598-9e70-a94f4d4bbe52") + self.assertEqual(profile["profile"]["kpp"], "772601001") + self.assertEqual(profile["sections"]["encumbrances"], "empty") + self.assertIn("ИНН", profile["rendered_sections"]["profile"]["text"]) + self.assertIn("tree", profile["rendered_sections"]["profile"]) + self.assertIn("АКЦИОНЕРНОЕ ОБЩЕСТВО ТЕСТ", profile["rendered_page"]["text"]) + self.assertIn( + {"label": "ИНН", "value": "6150061451"}, + profile["rendered_sections"]["profile"]["attributes"], + ) + + def test_profile_keeps_new_unknown_sections_without_schema_changes(self): + html = PROFILE_HTML + ( + '
' + '
Новый атрибут
Полное значение
' + ) + profile = parse_profile_html( + html, + url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", + inn="6150061451", + ogrn="1096100003625", + ) + self.assertIn("Полное значение", profile["rendered_page"]["text"]) + self.assertIn("Полное значение", profile["rendered_sections"]["additional-attributes"]["text"]) + + def test_related_message_keeps_all_rendered_fields_and_links(self): + page = parse_related_page_html( + '
' + '
Аудируемое лицо
АО ЭКОС
' + '
Мнение аудитора
Полный текст заключения
' + 'Документ
', + url="https://fedresurs.ru/sfactmessages/b2beb243-fffb-40c6-8b10-6c890ba59578", + page_type="message", + ) + self.assertIn("Полный текст заключения", page["text"]) + self.assertEqual(page["links"][0]["url"], "https://fedresurs.ru/documents/a.pdf") + self.assertIn( + {"label": "Мнение аудитора", "value": "Полный текст заключения"}, + page["attributes"], + ) + self.assertNotIn("_ngcontent", str(page["tree"])) + + def test_only_follows_discovered_same_company_pages(self): + profile = { + "fedresurs_guid": "e1661e0e-d217-4598-9e70-a94f4d4bbe52", + "rendered_sections": { + "publications": {"links": [ + {"url": "https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52/publications"}, + {"url": "https://fedresurs.ru/companies/00000000-0000-0000-0000-000000000000/publications"}, + {"url": "https://evil.example/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52/publications"}, + ]}, + }, + } + self.assertEqual(_related_company_urls(profile), [ + ("publications", "https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52/publications") + ]) + + def test_challenge_and_partial_page_are_upstream_errors(self): + with self.assertRaises(FedresursUpstreamError): + parse_profile_html("Проверка Qrator CAPTCHA", url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", inn="6150061451", ogrn="1096100003625") + with self.assertRaises(FedresursUpstreamError): + parse_profile_html(PROFILE_HTML.replace('page-item-name="general-info"', 'data-missing="1"'), url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", inn="6150061451", ogrn="1096100003625") + with self.assertRaisesRegex(FedresursUpstreamError, "upstream_access_blocked"): + parse_search_html( + "
403 Error
Forbidden
" + "

Access to fedresurs.ru is forbidden.

", + inn="6150061451", ogrn="1096100003625", + ) + + def test_script_mentions_of_challenge_do_not_hide_valid_public_page(self): + profile = parse_profile_html( + PROFILE_HTML + '', + url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", + inn="6150061451", + ogrn="1096100003625", + ) + self.assertEqual(profile["profile"]["inn"], "6150061451") + + def test_wait_detects_late_403_instead_of_reporting_timeout(self): + class BlockedBrowser: + page_source = ( + "
403 Error
" + "

Access to fedresurs.ru is forbidden.

" + ) + + def find_elements(self, *_args): + return [] + + with self.assertRaisesRegex(FedresursUpstreamError, "upstream_access_blocked"): + _wait_public_element(BlockedBrowser(), "css selector", "input") diff --git a/tests/apps/parsers/test_fedresurs_snapshot.py b/tests/apps/parsers/test_fedresurs_snapshot.py new file mode 100644 index 0000000..6fb5ae9 --- /dev/null +++ b/tests/apps/parsers/test_fedresurs_snapshot.py @@ -0,0 +1,79 @@ +from unittest.mock import patch + +from apps.parsers.fedresurs_company_profile import FedresursUpstreamError +from apps.parsers.fedresurs_snapshot import refresh_fedresurs_company_profiles +from django.test import TestCase, override_settings +from django.utils import timezone +from organizations.models import Organization, OrganizationSourceRecord + + +def profile(guid, inn, ogrn): + return { + "fedresurs_guid": guid, + "source_url": f"https://fedresurs.ru/companies/{guid}", + "profile": { + "full_name": "АО Тест", + "registration_date": "2020-01-01", + "status": {"is_active": True}, + "inn": inn, + "ogrn": ogrn, + }, + "sections": {"bankruptcy": "empty"}, + } + + +@override_settings( + FEDRESURS_COMPANY_PROFILE_ENABLED=True, + FEDRESURS_MIN_INTERVAL_SECONDS=0, + FEDRESURS_MAX_RETRIES=0, +) +class FedresursSnapshotTest(TestCase): + def setUp(self): + self.first = Organization.objects.create( + name="АО Первый", inn="6150061451", ogrn="1096100003625", + okpo="00084008", directory_imported_at=timezone.now(), + opk_registry_membership=True, + ) + self.second = Organization.objects.create( + name="АО Второй", inn="1234567890", ogrn="1027700132195", + okpo="00084009", directory_imported_at=timezone.now(), + opk_registry_membership=True, + ) + + @patch("apps.parsers.fedresurs_snapshot.public_browser") + @patch("apps.parsers.fedresurs_snapshot.lookup_company") + def test_complete_batch_publishes_and_targeted_refresh_preserves_other_records(self, lookup, browser): + browser.return_value.__enter__.return_value = object() + lookup.side_effect = lambda driver, *, inn, ogrn: ( + "match", profile( + "e1661e0e-d217-4598-9e70-a94f4d4bbe52" if inn == self.first.inn + else "a1661e0e-d217-4598-9e70-a94f4d4bbe52", inn, ogrn, + ) + ) + _, result = refresh_fedresurs_company_profiles(load_batch=1) + self.assertEqual(result.published, 2) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 2) + lookup.return_value = ("no_match", None) + lookup.side_effect = None + _, targeted = refresh_fedresurs_company_profiles( + load_batch=2, mode="targeted", organization_uids=[str(self.first.uid)] + ) + self.assertEqual(targeted.quarantined, 1) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 1) + + @patch("apps.parsers.fedresurs_snapshot.public_browser") + @patch("apps.parsers.fedresurs_snapshot.lookup_company") + def test_technical_failure_keeps_last_published_snapshot(self, lookup, browser): + browser.return_value.__enter__.return_value = object() + lookup.side_effect = lambda driver, *, inn, ogrn: ( + "match", profile( + "e1661e0e-d217-4598-9e70-a94f4d4bbe52" if inn == self.first.inn + else "a1661e0e-d217-4598-9e70-a94f4d4bbe52", inn, ogrn, + ) + ) + refresh_fedresurs_company_profiles(load_batch=1) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 2) + lookup.side_effect = FedresursUpstreamError("upstream_challenge") + with self.assertRaises(FedresursUpstreamError): + refresh_fedresurs_company_profiles(load_batch=2) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 2) diff --git a/tests/apps/parsers/test_integrated_structure_workbook.py b/tests/apps/parsers/test_integrated_structure_workbook.py new file mode 100644 index 0000000..3776dd1 --- /dev/null +++ b/tests/apps/parsers/test_integrated_structure_workbook.py @@ -0,0 +1,282 @@ +from io import BytesIO +from unittest.mock import patch + +from apps.parsers.integrated_structure_workbook import ( + IntegratedWorkbookError, + parse_integrated_workbook, +) +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import SimpleTestCase, TestCase, override_settings +from django.utils import timezone +from openpyxl import Workbook +from organizations.models import Organization, OrganizationSourceRecord +from rest_framework.test import APIClient + +from tests.apps.user.factories import UserFactory + + +def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137): + workbook = Workbook() + stages = workbook.active + stages.title = "is_stages" + stages.append(["okpo", "stage_id", "stage_date", "stage_prim"]) + stages.append([okpo, 1137, "2026-09-01", "Первое"]) + stages.append([okpo, 1245, "2026-09-02", None]) + documents = workbook.create_sheet("is_stages_docs") + documents.append( + [ + "okpo", "stage_id", "stage_date", "doc_id", "doc_name", + "doc_type", "doc_n", "doc_date", "dsp", "have_text", + "cons_local", "cons_web", + ] + ) + documents.append( + [ + okpo, document_stage, "2026-09-01", doc_id, "Документ", + "Приказ", "001", "2026-09-01", 0, 1, "local://not-for-browser", + "https://online.consultant.ru/example", + ] + ) + stream = BytesIO() + workbook.save(stream) + return stream.getvalue() + + +class IntegratedWorkbookTest(SimpleTestCase): + def test_parses_relations_and_preserves_zero_padded_okpo(self): + result = parse_integrated_workbook(BytesIO(workbook_bytes())) + self.assertEqual(result.counts, (1, 2, 1, 1, 1)) + self.assertEqual(result.structures["00084008"].events[0].external_id, "1137") + self.assertEqual(result.structures["00084008"].events[1].documents, []) + self.assertEqual(result.documents["93"].number, "001") + self.assertEqual( + result.documents["93"].consultant_web_url, + "https://online.consultant.ru/example", + ) + + def test_rejects_numeric_okpo_because_leading_zeroes_are_unrecoverable(self): + with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_okpo"): + parse_integrated_workbook(BytesIO(workbook_bytes(okpo=84008))) + + def test_rejects_orphan_document_relation(self): + with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"): + parse_integrated_workbook( + BytesIO(workbook_bytes(document_stage=9999)) + ) + + def test_rejects_non_allowlisted_document_url(self): + value = workbook_bytes() + workbook = Workbook() + # Use an ordinary generated fixture rather than modifying ZIP internals. + from openpyxl import load_workbook + + workbook = load_workbook(BytesIO(value)) + workbook["is_stages_docs"]["L2"] = "https://attacker.example/file" + stream = BytesIO() + workbook.save(stream) + with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_consultant_url"): + parse_integrated_workbook(BytesIO(stream.getvalue())) + + def test_rejects_duplicate_stage(self): + from openpyxl import load_workbook + + workbook = load_workbook(BytesIO(workbook_bytes())) + workbook["is_stages"].append(["00084008", 1137, "2026-09-01", "duplicate"]) + stream = BytesIO() + workbook.save(stream) + with self.assertRaisesRegex(IntegratedWorkbookError, "duplicate_stage"): + parse_integrated_workbook(BytesIO(stream.getvalue())) + + +class IntegratedStructureImportTest(TestCase): + def setUp(self): + from tempfile import TemporaryDirectory + + self.private = TemporaryDirectory() + self.addCleanup(self.private.cleanup) + self.settings = override_settings( + PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name, + MEDIA_ROOT=self.private.name + "/public", + ) + self.settings.enable() + self.addCleanup(self.settings.disable) + Organization.objects.create( + name="АО Тест", inn="1234567890", ogrn="1027700132195", + okpo="00084008", directory_imported_at=timezone.now(), + ) + + def test_atomic_publish_relations_attachment_and_idempotency(self): + from apps.parsers.integrated_structures import import_integrated_structures + from apps.parsers.models import ( + IntegratedStructureAttachment, + IntegratedStructureDocument, + IntegratedStructureEvent, + IntegratedStructureEventDocument, + ) + + workbook = workbook_bytes() + attachment = BytesIO(b"{\\rtf1\\ansi document}") + attachment.name = "000093.rtf" + first = import_integrated_structures( + workbook=BytesIO(workbook), original_name="test.xlsx", + attachments=[attachment], uploaded_by_id=None, + ) + second = import_integrated_structures( + workbook=BytesIO(workbook), original_name="test.xlsx", + attachments=[attachment], uploaded_by_id=None, + ) + self.assertEqual(first[0].uid, second[0].uid) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1) + self.assertEqual(IntegratedStructureEvent.objects.count(), 2) + self.assertEqual(IntegratedStructureDocument.objects.count(), 1) + self.assertEqual(IntegratedStructureEventDocument.objects.count(), 1) + self.assertEqual(IntegratedStructureAttachment.objects.count(), 1) + + def test_text_document_id_keeps_leading_zeroes_for_attachment(self): + from apps.parsers.integrated_structures import import_integrated_structures + from apps.parsers.models import IntegratedStructureDocument + + attachment = BytesIO(b"{\\rtf1\\ansi document}") + attachment.name = "000093.rtf" + import_integrated_structures( + workbook=BytesIO(workbook_bytes(doc_id="000093")), + original_name="test.xlsx", + attachments=[attachment], + ) + document = IntegratedStructureDocument.objects.get(external_id="000093") + self.assertEqual(document.attachment_filename, "000093.rtf") + self.assertIsNotNone(document.attachment) + + def test_invalid_batch_retains_last_snapshot(self): + from apps.parsers.integrated_structures import import_integrated_structures + + valid = workbook_bytes() + import_integrated_structures( + workbook=BytesIO(valid), original_name="test.xlsx", attachments=[] + ) + with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"): + import_integrated_structures( + workbook=BytesIO(workbook_bytes(document_stage=9999)), + original_name="bad.xlsx", attachments=[], + ) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1) + + def test_failed_publication_preserves_rejected_artifact_and_failed_log(self): + from apps.parsers.integrated_structures import import_integrated_structures + from apps.parsers.models import ParserLoadLog, ParserSourceArtifact + + with patch( + "apps.parsers.integrated_structures._publish_relations", + side_effect=RuntimeError("storage_failure"), + ), self.assertRaisesRegex(RuntimeError, "storage_failure"): + import_integrated_structures( + workbook=BytesIO(workbook_bytes()), + original_name="test.xlsx", + attachments=[], + ) + artifact = ParserSourceArtifact.objects.get(source="integrated_structure_documents") + load_log = ParserLoadLog.objects.get(source="integrated_structure_documents") + self.assertEqual(artifact.status, ParserSourceArtifact.Status.REJECTED) + self.assertEqual(load_log.status, ParserLoadLog.Status.FAILED) + self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 0) + + def test_export_has_three_relational_sheets_without_binary(self): + from tempfile import TemporaryDirectory + + from apps.parsers.integrated_structures import import_integrated_structures + from openpyxl import load_workbook + from organizations.source_record_export import ( + build_source_record_export_artifacts, + ) + + attachment = BytesIO(b"{\\rtf1\\ansi document}") + attachment.name = "000093.rtf" + import_integrated_structures( + workbook=BytesIO(workbook_bytes()), original_name="test.xlsx", + attachments=[attachment], + ) + with TemporaryDirectory() as directory: + generation = build_source_record_export_artifacts(export_directory=directory) + artifact = next( + item for item in generation.artifacts + if item.source_group == "integrated_structure_documents" and item.file_format == "xlsx" + ) + workbook = load_workbook(artifact.path, read_only=True) + self.assertEqual(workbook.sheetnames, ["structures", "events", "event_documents"]) + self.assertEqual(sum(1 for _ in workbook["structures"].iter_rows()), 2) + self.assertEqual(sum(1 for _ in workbook["events"].iter_rows()), 3) + self.assertEqual(sum(1 for _ in workbook["event_documents"].iter_rows()), 2) + self.assertNotIn(b"{\\rtf1", artifact.path.read_bytes()) + workbook.close() + + def test_detail_and_authorized_attachment_stream(self): + from apps.parsers.integrated_structures import import_integrated_structures + + attachment = BytesIO(b"{\\rtf1\\ansi document}") + attachment.name = "000093.rtf" + import_integrated_structures( + workbook=BytesIO(workbook_bytes()), original_name="test.xlsx", + attachments=[attachment], + ) + record = OrganizationSourceRecord.objects.get(source="integrated_structures") + path = f"/api/v2/organization-source-records/{record.uid}/" + download = path + "documents/93/attachment/" + client = APIClient() + self.assertEqual(client.get(download).status_code, 401) + with override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True): + self.assertEqual(client.get(download).status_code, 401) + client.force_authenticate(UserFactory.create_user()) + detail = client.get(path) + self.assertEqual(detail.status_code, 200) + payload = detail.data["payload"] + self.assertEqual(payload["structure"]["events_count"], 2) + self.assertEqual(len(payload["events"]), 2) + self.assertEqual(payload["events"][1]["documents"][0]["attachment"]["filename"], "000093.rtf") + response = client.get(download) + self.assertEqual(response.status_code, 200) + self.assertEqual(b"".join(response.streaming_content), b"{\\rtf1\\ansi document}") + + def test_upload_requires_admin_and_reports_missing_rtf(self): + user = UserFactory.create_user() + admin = UserFactory.create_superuser() + client = APIClient() + url = "/api/v1/parsers/upload/integrated_structure_documents/" + client.force_authenticate(user) + response = client.post( + url, + {"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())}, + format="multipart", + ) + self.assertEqual(response.status_code, 403) + client.force_authenticate(admin) + response = client.post( + url, + {"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())}, + format="multipart", + ) + self.assertEqual(response.status_code, 200) + record = OrganizationSourceRecord.objects.get(source="integrated_structures") + client.force_authenticate(user) + missing = client.get( + f"/api/v2/organization-source-records/{record.uid}/documents/93/attachment/" + ) + self.assertEqual(missing.status_code, 409) + self.assertEqual(missing.data["errors"][0]["code"], "attachment_unavailable") + + def test_unexpected_upload_failure_finishes_background_job(self): + from apps.core.models import BackgroundJob, JobStatus + + client = APIClient() + client.raise_request_exception = False + client.force_authenticate(UserFactory.create_superuser()) + with patch( + "apps.parsers.integrated_structures.import_integrated_structures", + side_effect=RuntimeError("storage_failure"), + ): + response = client.post( + "/api/v1/parsers/upload/integrated_structure_documents/", + {"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())}, + format="multipart", + ) + self.assertEqual(response.status_code, 500) + self.assertEqual(BackgroundJob.objects.get().status, JobStatus.FAILURE) diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index b10d442..0d2f3ef 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -95,6 +95,8 @@ class SourceCardServiceUnitTest(TestCase): "sme-support-recipients-registry", "budget-process-registry", "sro-membership-check", + "integrated-structure-documents", + "fedresurs-company-profile", ], ) self.assertEqual( @@ -115,6 +117,8 @@ class SourceCardServiceUnitTest(TestCase): "Реестр субъектов МСП — получателей поддержки", "Реестр участников бюджетного процесса", "Проверка членства в СРО", + "Документы интегрированных структур", + "Федресурс: сведения о юридических лицах", ], )