diff --git a/.env.prod.example b/.env.prod.example index 0d1dedf..0243de8 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -40,6 +40,10 @@ COLLECTSTATIC_ON_MIGRATE=0 BACKUP_ENCRYPTION_KEY=a2tra2tra2tra2tra2tra2tra2tra2tra2tra2s BACKUP_KEY_ID=default BACKUP_EXPORT_DIRECTORY=/app/media/backups +SOURCE_RECORD_EXPORT_DIRECTORY=/app/media/source-record-exports +SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP=2 +SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE=100000 +SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS=300 STATE_CORP_EXCHANGE_URL= STATE_CORP_EXCHANGE_TOKEN= diff --git a/README.md b/README.md index 5c649b6..3031d26 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,6 @@ Backend-сервис на Django/DRF для сбора, хранения и вы - `DJANGO_SETTINGS_MODULE`: `settings.dev` (локально/dev) или `settings.production` (prod). - `POSTGRES_*`: доступ к PostgreSQL. - `REDIS_CACHE_URL`, `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`: Redis/Celery. -- `CHECKO_API_KEY`: ключ API Checko. - `ZAKUPKI_TOKEN`: токен SOAP API ЕИС закупок. - `COLLECTSTATIC_ON_MIGRATE`: `1`/`0`. - `STARTUP_CHECKS_ENABLED`: fail-fast проверки DB/Redis перед стартом runtime-процессов. @@ -230,6 +229,20 @@ Celery-задачи (основные): - успешные: `input/fns/processed/` - ошибки: `input/fns/failed/` +Парсер поддерживает отчеты как с отдельной колонкой `Код`, так и стандартные +файлы, где после наименования строки сразу идут пары `Начало`/`Конец`. Для +восстановления ранее успешно обработанных отчетов без нормализованных +финансовых строк используется идемпотентная команда: + +```bash +# Проверка без записи +uv run python src/manage.py restore_fns_financial_lines --dry-run + +# Восстановление всех пустых отчетов или одной организации +uv run python src/manage.py restore_fns_financial_lines +uv run python src/manage.py restore_fns_financial_lines --ogrn 1187700006273 +``` + ## Тесты - Все тесты проекта хранятся только в `ROOT_DIR/tests`. diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 82aa951..82d57ee 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -104,6 +104,7 @@ services: memswap_limit: 3g volumes: - ./input:/app/input + - ./media:/app/media command: ["/app/docker/scripts/start-celery-worker.sh"] celery_beat: diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index fdad9d5..8128752 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -64,6 +64,7 @@ services: volumes: - ./logs:/app/logs - ./input:/app/input + - ./media:/app/media command: ["/app/docker/scripts/start-celery-worker.sh"] celery_beat: diff --git a/docker/Dockerfile b/docker/Dockerfile index 48db094..dcded6f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -103,10 +103,13 @@ ENV PATH="/app/.venv/bin:${PATH}" \ BACKUP_ENCRYPTION_KEY= \ BACKUP_KEY_ID=default \ BACKUP_EXPORT_DIRECTORY=/app/media/backups \ + SOURCE_RECORD_EXPORT_DIRECTORY=/app/media/source-record-exports \ + SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE=100000 \ + SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS=300 \ STATE_CORP_EXCHANGE_URL= \ STATE_CORP_EXCHANGE_TOKEN= \ STATE_CORP_EXCHANGE_KEY_ID=state-corp-shared-token \ - STATE_CORP_EXCHANGE_TIMEOUT_SECONDS=60 + STATE_CORP_EXCHANGE_TIMEOUT_SECONDS=300 USER appuser diff --git a/docs/parser-external-access-note-ru.md b/docs/parser-external-access-note-ru.md index c67d634..68d0d7c 100644 --- a/docs/parser-external-access-note-ru.md +++ b/docs/parser-external-access-note-ru.md @@ -21,13 +21,13 @@ | ЕИС закупки: HTTP fallback | `https://zakupki.gov.ru/opendata/download/notifications/{region}/{year}/...` | ZIP-архивы с XML-файлами закупок, если SOAP-токен не используется или передана прямая ссылка. | | ЕИС/FAS generic-источники | `https://zakupki.gov.ru/epz/order/extendedsearch/results.html`, `https://zakupki.gov.ru/epz/orderclause/search/results.html`, `https://zakupki.gov.ru/epz/contract/search/results.html`, `https://zakupki.gov.ru/epz/dishonestsupplier/search/results.html`, `https://fas.gov.ru/pages/activity/reestr-uridicheskih-lic` | HTML-страницы официальных реестров. Парсер извлекает карточки/таблицы: закупки 44-ФЗ, закупки 223-ФЗ, контракты, недобросовестные поставщики, сведения ФАС по ГОЗ. | | ФНС: бухгалтерская отчетность | Автоматического HTTP-скачивания с ФНС в текущем коде не найдено. В каталоге источников указан справочный URL `https://bo.nalog.gov.ru/advanced-search/organizations/search?...` | Обрабатываются локально загруженные или положенные в папку `input/fns` файлы `fin_{id}_{ogrn}.xlsx`, а также ZIP-архивы с такими файлами. Из Excel берутся строки форм N 1, 2, 3, 4, 6 бухгалтерской отчетности. | -| КАД Арбитр через Checko | Официальный источник в каталоге: `https://kad.arbitr.ru/`; фактический lookup в коде: `https://api.checko.ru/v2/legal-cases` | JSON-ответы по арбитражным делам для активных организаций из внутренних реестров. В запрос передаются ИНН/ОГРН. В payload сохраняются номер дела, суд, тип, статус, даты, суммы, стороны и ссылка на карточку. | -| Федресурс/ЕФРСБ | `https://bankrot.fedresurs.ru/`; fallback: `https://api.checko.ru/v2/company` | Официальный источник обрабатывается как HTML/структурированная выгрузка. При недоступности портала используется Checko: по ИНН/ОГРН организации берутся сведения о банкротных сообщениях из JSON. | +| КАД Арбитр через внешний сервис данных | Официальный источник в каталоге: `https://kad.arbitr.ru/`; фактический lookup выполняется через служебный API внешнего сервиса | JSON-ответы по арбитражным делам для активных организаций из внутренних реестров. В запрос передаются ИНН/ОГРН. В payload сохраняются номер дела, суд, тип, статус, даты, суммы, стороны и ссылка на карточку. | +| Федресурс/ЕФРСБ | `https://bankrot.fedresurs.ru/`; fallback выполняется через служебный API внешнего сервиса | Официальный источник обрабатывается как HTML/структурированная выгрузка. При недоступности портала по ИНН/ОГРН организации запрашиваются сведения о банкротных сообщениях из JSON. | | ФСТЭК | `https://reestr.fstec.ru/reg3` и найденные на странице ссылки вида `module=rfiles` или `/uploads/reg...` | HTML-страница реестра, затем CSV/файловая выгрузка, если ссылка найдена. Для этого источника в коде отключена SSL-верификация. | | Вакансии: Работа России | Клиент использует `http://opendata.trudvsem.ru/api/v1/vacancies`, `http://opendata.trudvsem.ru/api/v1/vacancies/company/inn/{inn}`; в каталоге источников указан `https://opendata.trudvsem.ru/api/v1/vacancies` | JSON-список вакансий, включая работодателя, ИНН/ОГРН при наличии, название вакансии, дату, зарплату, статус, ссылку. | | Вакансии: HeadHunter | `https://api.hh.ru/vacancies` | JSON-список вакансий. Поиск выполняется по региону и/или тексту, для организаций без поиска по ИНН используется нормализованное название. | | Вакансии: SuperJob | `https://api.superjob.ru/2.0/vacancies/` | JSON-список вакансий. Используется только если задан `SUPERJOB_APP_ID`; ключ передается в заголовке `X-Api-App-Id`. | -| Checko: контракты и проверки по организациям | `https://api.checko.ru/v2/contracts`, `https://api.checko.ru/v2/inspections` | JSON-данные по контрактам и проверкам для активных организаций из внутренних реестров. В запрос передаются ИНН/ОГРН, API-ключ передается параметром `key`. | +| Внешний сервис данных: контракты и проверки по организациям | Служебные API контрактов и проверок | JSON-данные по контрактам и проверкам для активных организаций из внутренних реестров. В запрос передаются ИНН/ОГРН, API-ключ передается параметром `key`. | | Proxy-Tools | `https://proxy-tools.com/api/v1/proxies` | Служебная загрузка списка RU-прокси для парсеров. Используется только при заданном `PROXY_TOOLS_API_KEY`; запрос идет с Bearer-токеном. | ## Форматы загружаемых данных @@ -51,7 +51,7 @@ - коды регионов; - ИНН/ОГРН организаций из внутренних активных реестров; - поисковая строка по названию организации для вакансий; -- служебные ключи API из окружения: `ZAKUPKI_TOKEN`, `CHECKO_API_KEY`, `SUPERJOB_APP_ID`, `PROXY_TOOLS_API_KEY`. +- служебные ключи API из окружения для ЕИС, внешнего сервиса данных, SuperJob и Proxy-Tools. Ключи в коде не захардкожены, берутся из переменных окружения. diff --git a/docs/source-record-export-matrix-ru.md b/docs/source-record-export-matrix-ru.md new file mode 100644 index 0000000..5e0f3a5 --- /dev/null +++ b/docs/source-record-export-matrix-ru.md @@ -0,0 +1,121 @@ +# Матрица файловых выгрузок источников + +## Пользовательский контракт + +Frontend отправляет администраторский +`POST /api/v2/organization-source-records/export-ticket/` с массивом `sources` +и выбранным `format`. В ответ он получает короткоживущий одноразовый ticket и +передаёт его обычной HTML-формой в +`POST /api/v2/organization-source-records/export-download/`. Поэтому браузер +сохраняет потоковый ZIP напрямую на диск, не удерживая весь архив как Blob в +JavaScript. Ticket передаётся в теле формы, не попадает в URL и после первого +запроса становится недействительным. + +Имя скачиваемого архива формируется по выбранным источникам и времени создания +запроса: `_YYYYMMDD_HHMMSS.zip`, а для нескольких источников их +имена соединяются через `__`. Это же имя возвращается в `file_name` при выдаче +ticket и затем используется в `Content-Disposition`. + +Совместимый администраторский +`POST /api/v2/organization-source-records/export/` по-прежнему сразу возвращает +тот же ZIP API-клиентам. Крупный XLSX может состоять из нескольких файлов +`*-part-001.xlsx`, `*-part-002.xlsx` и далее. + +При скачивании endpoint не читает таблицы записей источников и не строит CSV, +XLSX или JSON заново. Он упаковывает файлы последнего полностью опубликованного +ночного поколения и сразу потоково отправляет ZIP без временной копии всего +архива. Поэтому `Content-Length` у ответа отсутствует. Если ни одного поколения +ещё нет, API отвечает `503` с кодом `source_export_not_ready`. + +Каждое поколение содержит только текущий календарный год в timezone сервиса. +Для записей с предметной датой год определяется по ней, для записей без такой +даты — по `created_at`, для финансовых отчётов — по году `financial_lines`. +Вложенные финансовые строки других лет исключаются. После смены года поколение +прошлого года не раздаётся: до первой успешной сборки нового года API отвечает +`503 source_export_not_ready`. + +## Матрица + +| Группа API | Файл | CSV | XLSX | JSON | +|---|---|:---:|:---:|:---:| +| `financial_indicators` | `financial-indicators` | — | — | да | +| `government_procurements` | `public-procurements` | да | да | да | +| `industrial_production` | `manufacturers-and-products` | да | да | да | +| `planned_inspections` | `planned-inspections` | да | да | да | +| `bankruptcy` | `bankruptcy-procedures` | да | да | да | +| `defense_suppliers` | `defense-unreliable-suppliers` | да | да | да | +| `arbitration` | `arbitration-cases` | да | да | да | +| `security_registries` | `information-security-registries` | да | да | да | +| `vacancies` | `labor-vacancies` | да | да | да | + +Итого формируется 25 логических артефактов: один JSON для финансовых показателей +и по три формата для остальных восьми групп. Физических файлов может быть +больше из-за разбиения крупных XLSX. Если финансовые показатели выбраны вместе +с другим форматом, в ZIP для них всё равно включается JSON. + +Все форматы начинают строку организации с полей `Наименование`, `ИНН`, `ОГРН`, +`КПП`, `ОКПО`, после которых следуют поля исходной записи и развёрнутого +`payload`. Все записи текущего года включаются в публичные файлы, а техническое +наименование внешнего поставщика нейтрализуется. Исходные значения в БД +сохраняются для работы интеграции и дедупликации. + +## Ночная генерация + +Celery Beat запускает +`organizations.tasks.refresh_source_record_export_artifacts` ежедневно в +`05:30 Europe/Moscow`, после ежедневного обновления organization sources в +`04:30`. + +Генератор: + +1. читает каждую группу из БД один раз без глобальной сортировки миллионов строк; + выборка текущего года использует функциональный индекс по году строковой + `record_date`, а записи без предметной даты — индекс `created_at`; +2. пишет compact JSON-массив на диск и использует его как готовый JSON без второй копии; +3. потоково создаёт CSV и XLSX без накопления всех строк в памяти; +4. разбивает XLSX по умолчанию по 100 000 строк на отдельные файлы, ограничивая временный XML и не превышая лимит Excel; +5. записывает размеры файлов, номера частей и календарный `export_year` в manifest; +6. атомарно переключает `current.json` только после готовности всей матрицы; +7. сохраняет текущее и предыдущее поколения по умолчанию. + +При ошибке незавершённое поколение удаляется, а download endpoint продолжает +отдавать предыдущую успешную версию. + +### Расчёт диска + +Атомарная публикация требует одновременно хранить уже опубликованные поколения +и одно новое поколение в staging. Минимальный запас под артефакты рассчитывается +как `(SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP + 1) * размер поколения`, плюс +рабочий запас файловой системы. Исторический снимок dev от 2026-08-03 до +ограничения по году занимал 16,55 ГБ (54 физических файла); актуальный размер +годового поколения нужно брать из `total_size` результата команды сборки. +Временная копия целого ZIP при скачивании не создаётся. + +На локальном снимке от 2026-08-04 поколение за 2026 год содержит 122 582 записи +и занимает 809 768 139 байт (772,3 MiB) для всей матрицы из 25 файлов. Архив со +всеми источниками оценивается в 410,1 MiB для JSON, 289,9 MiB для CSV и +72,3 MiB для XLSX; ZIP использует `ZIP_STORED`, поэтому к сумме файлов +добавляется только небольшой служебный overhead. + +## Хранение и первый запуск + +Каталог задаётся через `SOURCE_RECORD_EXPORT_DIRECTORY`, по умолчанию — +`media/source-record-exports`. Он должен быть общим read-write volume для web и +Celery worker. В Docker Compose используется `./media:/app/media`. + +После первого развёртывания готовое поколение можно создать сразу, не ожидая +ночного расписания: + +```bash +PYTHONPATH=src uv run python src/manage.py build_source_record_exports +``` + +Доступные настройки: + +| Настройка | Значение по умолчанию | Назначение | +|---|---:|---| +| `SOURCE_RECORD_EXPORT_DIRECTORY` | `media/source-record-exports` | Общий каталог артефактов | +| `SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP` | `2` | Число сохраняемых успешных поколений | +| `SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS` | `21600` | TTL распределённой блокировки Celery | +| `SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE` | `100000` | Максимум строк данных в одной XLSX-части | +| `SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS` | `300` | Срок действия одноразового browser-download ticket | diff --git a/src/apps/core/models.py b/src/apps/core/models.py index 9b4cb7c..b258920 100644 --- a/src/apps/core/models.py +++ b/src/apps/core/models.py @@ -169,12 +169,16 @@ class BackgroundJob(TimestampMixin, models.Model): self.status = JobStatus.SUCCESS self.progress = 100 self.result = result + self.error = "" + self.traceback = "" self.completed_at = timezone.now() self.save( update_fields=[ "status", "progress", "result", + "error", + "traceback", "completed_at", "updated_at", ] diff --git a/src/apps/core/services.py b/src/apps/core/services.py index 1d2cd73..54c5821 100644 --- a/src/apps/core/services.py +++ b/src/apps/core/services.py @@ -752,21 +752,21 @@ class BackgroundJobService(BaseReadOnlyService): cls, *, max_age_minutes: int, + pending_max_age_minutes: int = 24 * 60, task_names: set[str] | None = None, meta_sources: set[str] | None = None, ) -> int: """Mark old active jobs as failed after worker restarts or hard kills.""" from apps.core.models import JobStatus - cutoff = timezone.now() - timedelta(minutes=max_age_minutes) - queryset = ( - cls.get_queryset() - .filter( - status__in=[JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY], - ) - .filter( - Q(started_at__isnull=False, started_at__lt=cutoff) - | Q(started_at__isnull=True, created_at__lt=cutoff) + now = timezone.now() + cutoff = now - timedelta(minutes=max_age_minutes) + pending_cutoff = now - timedelta(minutes=pending_max_age_minutes) + queryset = cls.get_queryset().filter( + Q(status=JobStatus.PENDING, created_at__lt=pending_cutoff) + | Q( + status__in=[JobStatus.STARTED, JobStatus.RETRY], + updated_at__lt=cutoff, ) ) if task_names: diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py index 57848ca..b95ec5f 100644 --- a/src/apps/exchange/services.py +++ b/src/apps/exchange/services.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import uuid from contextlib import suppress from typing import Any @@ -35,13 +36,14 @@ class ExchangeConnectionService: @classmethod def test_connection_payload(cls, **payload) -> dict[str, str]: - """Проверить подключение и структуру без сохранения в БД.""" + """Проверить доступность PostgreSQL без сохранения в БД.""" connection = ExchangeConnection(is_active=False, **payload) - cls.validate_saved_connection(connection) + alias = cls.test_connection(connection) + cls._cleanup_alias(alias) return { "status": "success", - "message": "Подключение проверено. Соединение и структура БД валидны.", + "message": "Подключение проверено. PostgreSQL доступен, логин и пароль верны.", } @classmethod @@ -300,7 +302,10 @@ class ExchangeConnectionService: @classmethod def _configure_alias(cls, connection: ExchangeConnection) -> str: - alias = f"exchange_target_{connection.id}" + connection_key = ( + connection.pk if connection.pk is not None else uuid.uuid4().hex + ) + alias = f"exchange_target_{connection_key}" config = { "ENGINE": "django.db.backends.postgresql", @@ -497,7 +502,7 @@ class ExchangeConnectionService: if not cls._requires_registry_organizations(models_to_copy): return models_to_copy - organization_model = django_apps.get_model("registers.Organization") + organization_model = django_apps.get_model("organizations.Organization") ordered_models = [organization_model, *models_to_copy] unique_models = [] diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index a6ecf75..3a7a904 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -164,7 +164,7 @@ class StateCorpExchangeService: resolved_url = str(target_url or settings.STATE_CORP_EXCHANGE_URL).strip() resolved_token = str(token or settings.STATE_CORP_EXCHANGE_TOKEN).strip() timeout_seconds = timeout_seconds or int( - getattr(settings, "STATE_CORP_EXCHANGE_TIMEOUT_SECONDS", 60) + getattr(settings, "STATE_CORP_EXCHANGE_TIMEOUT_SECONDS", 300) ) if not resolved_url: @@ -785,6 +785,10 @@ class StateCorpExchangeService: if not case_number or decision_date is None: continue + claim_amount = cls._serialize_decimal(record.amount) + if claim_amount is None: + claim_amount = str(payload.get("claim_amount") or "").strip() or None + items.append( { "organization_inn": cls._digits(record.inn), @@ -795,6 +799,7 @@ class StateCorpExchangeService: ).strip(), "status": str(payload.get("status") or record.status or "").strip(), "decision_date": decision_date.isoformat(), + "claim_amount": claim_amount, } ) return items diff --git a/src/apps/exchange/views.py b/src/apps/exchange/views.py index fbda1d9..1a2d197 100644 --- a/src/apps/exchange/views.py +++ b/src/apps/exchange/views.py @@ -90,8 +90,8 @@ class ExchangeConnectionTestView(APIView): tags=[EXCHANGE_TAG], operation_summary="Проверить подключение", operation_description=( - "Проверяет подключение и структуру целевой БД без сохранения " - "настроек подключения." + "Проверяет доступность PostgreSQL и учетные данные без сохранения " + "настроек подключения. Проверка структуры выполняется при сохранении." ), request_body=ExchangeConnectionCreateSerializer, responses={ diff --git a/src/apps/parsers/checko_collection.py b/src/apps/parsers/checko_collection.py index 033740f..eddd0b7 100644 --- a/src/apps/parsers/checko_collection.py +++ b/src/apps/parsers/checko_collection.py @@ -1,4 +1,4 @@ -"""Quota-safe monthly claims for organization lookups in Checko.""" +"""Quota-safe monthly claims for external organization lookups.""" from datetime import date, datetime @@ -24,7 +24,7 @@ def claim_monthly_collection( source: str, at: date | datetime | None = None, ) -> CheckoCollectionAttempt | None: - """Atomically claim this month's only allowed Checko collection attempt.""" + """Atomically claim this month's only allowed external collection attempt.""" attempt, created = CheckoCollectionAttempt.objects.get_or_create( organization_id=organization_id, source=source, diff --git a/src/apps/parsers/clients/checko/client.py b/src/apps/parsers/clients/checko/client.py index d51833b..5a8c45e 100644 --- a/src/apps/parsers/clients/checko/client.py +++ b/src/apps/parsers/clients/checko/client.py @@ -346,14 +346,18 @@ class CheckoClient: # Preserve the existing classification for non-quota HTTP # errors while still recovering Checko quota metadata. pass - logger.error("Checko HTTP request failed with status=%s", e.status_code) + logger.error( + "External provider HTTP request failed with status=%s", e.status_code + ) raise CheckoConnectionError( - "Checko API request failed", + "External provider API request failed", url=e.url, ) from e except Exception as e: logger.error("Connection error: %s", e) - raise CheckoConnectionError(f"Failed to connect to Checko API: {e}") from e + raise CheckoConnectionError( + f"Failed to connect to external provider API: {e}" + ) from e self._raise_api_error(data) diff --git a/src/apps/parsers/clients/fns/line_codes.py b/src/apps/parsers/clients/fns/line_codes.py new file mode 100644 index 0000000..20b6891 --- /dev/null +++ b/src/apps/parsers/clients/fns/line_codes.py @@ -0,0 +1,271 @@ +"""Canonical line-code resolution for FNS workbooks without a code column.""" + +from __future__ import annotations + +import re + +_DASH_PATTERN = re.compile(r"[‐‑‒–—−]") + + +def normalize_fns_line_name(value: object) -> str: + """Normalize workbook labels without losing their accounting meaning.""" + normalized = str(value or "").strip().casefold().replace("ё", "е") + normalized = _DASH_PATTERN.sub("-", normalized) + return " ".join(normalized.split()) + + +CODELESS_FNS_LINE_CODES: dict[str, dict[str, str]] = { + "1": { + "баланс": "1600", + "дебиторская задолженность": "1230", + "денежные средства и денежные эквиваленты": "1250", + "добавочный капитал (без накопленной дооценки)": "1350", + "добавочный капитал (без переоценки)": "1350", + "долгосрочные заемные средства": "1410", + "доходные вложения в материальные ценности": "1160", + "доходы будущих периодов": "1530", + "другие долгосрочные обязательства": "1450", + "другие краткосрочные обязательства": "1550", + "запасы": "1210", + "инвестиционная недвижимость": "1160", + "капитал и резервы": "1300", + "краткосрочная кредиторская задолженность": "1520", + "краткосрочные заемные средства": "1510", + "кредиторская задолженность": "1520", + "материальные внеоборотные активы": "1150", + "материальные поисковые активы": "1140", + "накопленная дооценка внеоборотных активов": "1340", + "ндс по приобретенным ценностям": "1220", + "нематериальные активы": "1110", + "нематериальные поисковые активы": "1130", + "нематериальные, финансовые и другие внеоборотные активы": "1170", + "нераспределенная прибыль (непокрытый убыток)": "1370", + "основные средства": "1150", + "отложенные налоговые активы": "1180", + "отложенные налоговые обязательства": "1420", + "переоценка внеоборотных активов": "1340", + "прочие внеоборотные активы": "1190", + "прочие долгосрочные обязательства": "1450", + "прочие краткосрочные обязательства": "1550", + "прочие оборотные активы": "1260", + "резервный капитал": "1360", + "результаты исследований и разработок": "1120", + "собственные акции, выкупленные у акционеров": "1320", + "собственные акции, принадлежащие обществу, задолженность акционеров по оплате акций": "1320", + "уставный капитал": "1310", + "финансовые вложения": "1170", + "финансовые вложения (за исключением денежных эквивалентов)": "1240", + "финансовые и другие оборотные активы": "1230", + }, + "2": { + "базовая прибыль (убыток) на акцию": "2900", + "валовая прибыль (убыток)": "2100", + "выручка": "2110", + "доходы от участия в других организациях": "2310", + "изменение отложенных налоговых активов": "2450", + "изменение отложенных налоговых обязательств": "2430", + "коммерческие расходы": "2210", + "налог на прибыль": "2410", + "налог на прибыль от операций, результат которых не включается в чистую прибыль (убыток)": "2530", + "налоги на прибыль (доходы)": "2410", + "отложенный налог на прибыль": "2412", + "постоянные налоговые обязательства (активы)": "2421", + "прибыль (убыток) до налогообложения": "2300", + "прибыль (убыток) от продаж": "2200", + "проценты к получению": "2320", + "проценты к уплате": "2330", + "прочее": "2460", + "прочие доходы": "2340", + "прочие расходы": "2350", + "разводненная прибыль (убыток) на акцию": "2910", + "расходы по обычной деятельности": "2120", + "результат от переоценки внеоборотных активов, не включаемый в чистую прибыль (убыток)": "2510", + "результат от прочих операций, не включаемый в чистую прибыль (убыток) периода": "2520", + "себестоимость продаж": "2120", + "совокупный финансовый результат периода": "2500", + "текущий налог на прибыль": "2411", + "управленческие расходы": "2220", + "чистая прибыль (убыток)": "2400", + }, + "3": { + "величина капитала": "3300", + "дивиденды": "3327", + "добавочный капитал": "3300", + "дополнительный выпуск акций": "3314", + "доходы на увеличение капитала": "3313", + "изменение добавочного капитала": "3330", + "изменение резервного капитала": "3340", + "нераспределенная прибыль (непокрытый убыток)": "3300", + "расходы на уменьшение капитала": "3323", + "резервный капитал": "3300", + "собственные акции": "3300", + "убыток": "3321", + "увеличение - всего": "3310", + "увеличение номинальной стоимости акций": "3315", + "уменьшение - всего": "3320", + "уменьшение количества акций": "3325", + "уменьшение номинальной стоимости акций": "3324", + "уставный капитал": "3300", + "чистая прибыль": "3311", + "чистые активы": "3600", + }, + "4": { + "арендные, лицензензионные, комиссионные и т.п. платежи": "4112", + "влияние изменений курса валют": "4490", + "возврат займов, продажа долг. ценных бумаг, прав требования": "4213", + "выкуп акций (долей) у собственников (участников)": "4321", + "выпуск акций, увеличение долей участия": "4313", + "выпуск долговых ценных бумаг": "4314", + "денежные вклады собственников (участников)": "4312", + "дивиденды и т.п. платежи по распределению прибыли": "4322", + "дивиденды, проценты по долг. финанс. вложениям и т.п. поступления от долевого участия в др. организациях": "4214", + "налог на прибыль организаций": "4124", + "оплата труда работников": "4122", + "остаток денежных средств и эквивалентов на конец периода": "4500", + "остаток денежных средств и эквивалентов на начало периода": "4450", + "перепродажа финансовых вложений": "4113", + "погашение (выкуп) долг. ценных бумаг, возврат кредитов и займов": "4323", + "получение кредитов и займов": "4311", + "поставщикам (подрядчикам) за сырье, материалы, работы": "4121", + "приобретение акций др. организаций, долей участия": "4222", + "приобретение долг. ценных бумаг, прав требования, предоставление займов": "4223", + "приобретение, создание, модернизация, реконструкция, подготовка к использованию внеоборотных активов": "4221", + "продажа акций др. организаций, долей участия": "4212", + "продажа внеоборотных активов (кроме финанс. вложений)": "4211", + "продажа продукции, товаров, работ и услуг": "4111", + "проценты по долг. обязательствам, вкл. в стоимость инвестиц. актива": "4224", + "проценты по долговым обязательствам": "4123", + "сальдо денежных потоков": "4400", + }, + "6": { + "взносы и иные целевые поступления": "6220", + "вступительные взносы": "6210", + "выплаты, не связанные с оплатой труда": "6322", + "добровольные имущ. взносы и пожертвования": "6230", + "иные мероприятия": "6313", + "на приобретение основных средств и иного имущества": "6330", + "на содержание организации": "6320", + "на целевые мероприятия": "6310", + "оплата труда (включая начисления)": "6321", + "остаток средств на конец периода": "6400", + "остаток средств на начало периода": "6100", + "прибыль от предпринимательской деятельности": "6240", + "прибыль от предпринимательской и иной деятельности": "6240", + "приобретение основных средств, инвентаря и др. имущества": "6330", + "проведение конференций, совещаний, семинаров и т.п.": "6312", + "прочeе": "6350", + "прочие": "6326", + "прочие поступления": "6250", + "расходы на содержание аппарата управления": "6320", + "расходы на целевые мероприятия": "6310", + "ремонт основных средств и др. имущества": "6325", + "служебные командировки и деловые поездки": "6323", + "содержание основных средств и др. имущества (кроме ремонта)": "6324", + "социальная и благотворительная помощь": "6311", + "целевые взносы": "6220", + "членские взносы": "6215", + }, +} + + +_FORM_1_SECTION_CODES: dict[str, dict[str, str]] = { + "внеоборотные активы": {"итого": "1100"}, + "оборотные активы": {"итого": "1200"}, + "капитал и резервы": {"итого": "1300"}, + "долгосрочные обязательства": { + "заемные средства": "1410", + "итого": "1400", + "оценочные обязательства": "1430", + "прочие обязательства": "1450", + }, + "краткосрочные обязательства": { + "заемные средства": "1510", + "итого": "1500", + "оценочные обязательства": "1540", + "прочие обязательства": "1550", + }, +} + +_FORM_4_FLOW_CODES: dict[str, dict[str, str]] = { + "текущих": { + "платежи - всего": "4120", + "поступления - всего": "4110", + "прочие платежи": "4129", + "прочие поступления": "4119", + "сальдо": "4100", + }, + "инвестиционных": { + "платежи - всего": "4220", + "поступления - всего": "4210", + "прочие платежи": "4229", + "прочие поступления": "4219", + "сальдо": "4200", + }, + "финансовых": { + "платежи - всего": "4320", + "поступления - всего": "4310", + "прочие платежи": "4329", + "прочие поступления": "4319", + "сальдо": "4300", + }, +} + + +def _resolve_form_1_code(*, label: str, section: str) -> str | None: + return _FORM_1_SECTION_CODES.get(section, {}).get(label) + + +def _resolve_form_3_code(*, label: str, direction: str) -> str | None: + if label == "переоценка имущества": + return {"увеличение - всего": "3312", "уменьшение - всего": "3322"}.get( + direction + ) + if label == "реорганизация юл": + return {"увеличение - всего": "3316", "уменьшение - всего": "3326"}.get( + direction + ) + return None + + +def _resolve_form_4_code(*, label: str, section: str) -> str | None: + for flow_name, codes in _FORM_4_FLOW_CODES.items(): + if flow_name in section: + return codes.get(label) + return None + + +def _resolve_form_6_code(*, label: str, funds: str) -> str | None: + if label != "итого": + return None + return {"поступило средств": "6200", "использовано средств": "6300"}.get(funds) + + +def resolve_codeless_fns_line_code( + *, + form_code: str, + line_name: object, + section_name: object = "", + change_direction: object = "", + funds_section: object = "", +) -> str | None: + """Resolve a standard accounting code from a code-less FNS row.""" + label = normalize_fns_line_name(line_name) + section = normalize_fns_line_name(section_name) + direction = normalize_fns_line_name(change_direction) + funds = normalize_fns_line_name(funds_section) + + if form_code == "1": + contextual_code = _resolve_form_1_code(label=label, section=section) + elif form_code == "3": + contextual_code = _resolve_form_3_code(label=label, direction=direction) + elif form_code == "4": + contextual_code = _resolve_form_4_code(label=label, section=section) + elif form_code == "6": + contextual_code = _resolve_form_6_code(label=label, funds=funds) + else: + contextual_code = None + + if contextual_code: + return contextual_code + + return CODELESS_FNS_LINE_CODES.get(form_code, {}).get(label) diff --git a/src/apps/parsers/clients/fns/parser.py b/src/apps/parsers/clients/fns/parser.py index c0bf2c6..96c2292 100644 --- a/src/apps/parsers/clients/fns/parser.py +++ b/src/apps/parsers/clients/fns/parser.py @@ -7,6 +7,10 @@ import re from pathlib import Path import openpyxl +from apps.parsers.clients.fns.line_codes import ( + normalize_fns_line_name, + resolve_codeless_fns_line_code, +) from apps.parsers.clients.fns.schemas import ParsedReport, ReportLine logger = logging.getLogger(__name__) @@ -92,13 +96,23 @@ class FNSExcelParser: lines: list[ReportLine] = [] - for sheet_name in workbook.sheetnames: - sheet = workbook[sheet_name] - sheet_lines = cls._parse_sheet(sheet) - lines.extend(sheet_lines) - logger.debug("Лист '%s': извлечено %d строк", sheet_name, len(sheet_lines)) + try: + for sheet_name in workbook.sheetnames: + sheet = workbook[sheet_name] + sheet_lines = cls._parse_sheet(sheet) + lines.extend(sheet_lines) + logger.debug( + "Лист '%s': извлечено %d строк", + sheet_name, + len(sheet_lines), + ) + finally: + workbook.close() - workbook.close() + if not lines: + raise FNSParserError( + f"Файл {file_path.name} не содержит распознанных строк отчетности" + ) logger.info( "Файл %s обработан: %d строк, годы %s, формы %s", @@ -117,6 +131,10 @@ class FNSExcelParser: current_form: str | None = None years: list[int] = [] header_row_found = False + value_column_start: int | None = None + current_section = "" + change_direction = "" + funds_section = "" for row in sheet.iter_rows(values_only=True): if not any(row): @@ -130,51 +148,145 @@ class FNSExcelParser: current_form = form_code years = cls._extract_years_from_row(row) header_row_found = False + value_column_start = None + current_section = "" + change_direction = "" + funds_section = "" logger.debug("Найдена форма %s, годы: %s", form_code, years) break + else: + form_code = None - # Пропускаем заголовочную строку с "Код", "Начало", "Конец" - is_header = len(row) > 1 and row[1] == "Код" - if current_form and not header_row_found and is_header: - header_row_found = True + if form_code is not None: continue - # Парсим строки данных - if current_form and header_row_found and years: - line_name = first_cell + detected_value_start = cls._detect_value_column_start(row) + if current_form and not header_row_found and detected_value_start: + header_row_found = True + value_column_start = detected_value_start + continue + + if not ( + current_form + and header_row_found + and years + and value_column_start is not None + ): + continue + + period_values = [] + for year_idx, year in enumerate(years): + col_start = value_column_start + year_idx * 2 + col_end = col_start + 1 + period_start = cls._parse_value( + row[col_start] if col_start < len(row) else None + ) + period_end = cls._parse_value( + row[col_end] if col_end < len(row) else None + ) + period_values.append((year, period_start, period_end)) + + normalized_line_name = normalize_fns_line_name(first_cell) + has_values = any( + period_start is not None or period_end is not None + for _, period_start, period_end in period_values + ) + if not has_values: + if normalized_line_name and not normalized_line_name.isdigit(): + current_section = normalized_line_name + if current_form == "3": + change_direction = "" + if current_form == "6" and normalized_line_name in { + "поступило средств", + "использовано средств", + }: + funds_section = normalized_line_name + continue + + if not normalized_line_name: + raise FNSParserError( + f"Лист {sheet.title}: строка формы {current_form} " + "содержит значения без наименования" + ) + + if current_form == "3" and normalized_line_name in { + "увеличение - всего", + "уменьшение - всего", + }: + change_direction = normalized_line_name + + row_change_direction = change_direction + if ( + current_form == "3" + and not row_change_direction + and normalized_line_name in {"переоценка имущества", "реорганизация юл"} + ): + row_change_direction = cls._infer_change_direction(period_values) + + if value_column_start == 2: line_code = str(row[1]).strip() if row[1] else "" + else: + line_code = ( + resolve_codeless_fns_line_code( + form_code=current_form, + line_name=normalized_line_name, + section_name=current_section, + change_direction=row_change_direction, + funds_section=funds_section, + ) + or "" + ) - # Пропускаем строки без кода или заголовки секций - if not line_code or not line_code.isdigit(): + if not line_code.isdigit(): + raise FNSParserError( + f"Лист {sheet.title}: не определен код строки " + f"формы {current_form} «{first_cell}»" + ) + + for year, period_start, period_end in period_values: + if period_start is None and period_end is None: continue - - # Извлекаем значения по годам - for year_idx, year in enumerate(years): - col_start = 2 + year_idx * 2 # Начало: 2, 4, 6, 8 - col_end = 3 + year_idx * 2 # Конец: 3, 5, 7, 9 - - period_start = cls._parse_value( - row[col_start] if col_start < len(row) else None + lines.append( + ReportLine( + form_code=current_form, + line_code=line_code, + line_name=first_cell, + year=year, + period_start=period_start, + period_end=period_end, ) - period_end = cls._parse_value( - row[col_end] if col_end < len(row) else None - ) - - # Добавляем строку только если есть хотя бы одно значение - if period_start is not None or period_end is not None: - lines.append( - ReportLine( - form_code=current_form, - line_code=line_code, - line_name=line_name, - year=year, - period_start=period_start, - period_end=period_end, - ) - ) + ) return lines + @staticmethod + def _detect_value_column_start(row: tuple) -> int | None: + """Return the first period-value column for supported FNS layouts.""" + second_cell = normalize_fns_line_name(row[1] if len(row) > 1 else "") + third_cell = normalize_fns_line_name(row[2] if len(row) > 2 else "") + if second_cell == "код": + return 2 + if second_cell == "начало" and third_cell == "конец": + return 1 + return None + + @staticmethod + def _infer_change_direction( + period_values: list[tuple[int, int | None, int | None]], + ) -> str: + """Infer an unlabelled capital change only from an unambiguous sign.""" + values = [ + value + for _, period_start, period_end in period_values + for value in (period_start, period_end) + if value not in (None, 0) + ] + if values and all(value > 0 for value in values): + return "увеличение - всего" + if values and all(value < 0 for value in values): + return "уменьшение - всего" + return "" + @classmethod def _extract_years_from_row(cls, row: tuple) -> list[int]: """Извлекает годы из строки заголовка формы.""" diff --git a/src/apps/parsers/clients/gisp/client.py b/src/apps/parsers/clients/gisp/client.py index 3b5da88..204ada9 100644 --- a/src/apps/parsers/clients/gisp/client.py +++ b/src/apps/parsers/clients/gisp/client.py @@ -2,6 +2,8 @@ from __future__ import annotations +import logging +import time from dataclasses import dataclass, field from typing import Any @@ -18,6 +20,8 @@ MAX_PAGE_SIZE = 100 MAX_PAGES = 100 MAX_RECORDS = 10_000 +logger = logging.getLogger(__name__) + class GispProductsClientError(HTTPClientError): """Ошибка клиента реестра продукции ГИСП.""" @@ -44,6 +48,8 @@ class GispProductsClient: max_records: int = DEFAULT_MAX_RECORDS proxies: list[str] | None = None timeout: int = 120 + max_retries: int = 2 + retry_backoff_seconds: float = 1.0 http_adapter: BaseAdapter | None = None _http_client: BaseHTTPClient | None = field(default=None, repr=False) @@ -54,6 +60,10 @@ class GispProductsClient: raise ValueError(f"max_pages must be between 1 and {MAX_PAGES}") if not 1 <= self.max_records <= MAX_RECORDS: raise ValueError(f"max_records must be between 1 and {MAX_RECORDS}") + if self.max_retries < 0: + raise ValueError("max_retries must be non-negative") + if self.retry_backoff_seconds < 0: + raise ValueError("retry_backoff_seconds must be non-negative") @property def http_client(self) -> BaseHTTPClient: @@ -129,16 +139,33 @@ class GispProductsClient: take: int, previous_total_count: int | None, ) -> tuple[list[Any], int | None]: - data = self.http_client.post_json( - PRODUCTS_ENDPOINT, - payload={ - "opt": { - "skip": offset, - "take": take, - "requireTotalCount": True, - } - }, - ) + payload = { + "opt": { + "skip": offset, + "take": take, + "requireTotalCount": True, + } + } + for attempt in range(self.max_retries + 1): + try: + data = self.http_client.post_json( + PRODUCTS_ENDPOINT, + payload=payload, + ) + break + except HTTPClientError as exc: + transient = exc.status_code is None or exc.status_code >= 500 + if not transient or attempt >= self.max_retries: + raise + delay = self.retry_backoff_seconds * (2**attempt) + logger.warning( + "GISP request failed temporarily (attempt %d/%d): %s", + attempt + 1, + self.max_retries + 1, + exc, + ) + if delay: + time.sleep(delay) if data.get("ok") is False: raise GispProductsClientError("GISP products API returned ok=false") items = data.get("items") diff --git a/src/apps/parsers/clients/proverki/client.py b/src/apps/parsers/clients/proverki/client.py index bfc0df7..1d64593 100644 --- a/src/apps/parsers/clients/proverki/client.py +++ b/src/apps/parsers/clients/proverki/client.py @@ -693,7 +693,7 @@ class ProverkiClient: return inspections - def _parse_xml_streaming( + def _parse_xml_streaming( # noqa: C901 self, content: bytes, progress_callback: Callable[[int, str], None] | None = None, @@ -704,52 +704,53 @@ class ProverkiClient: Использует iterparse для обработки файла по элементам, не загружая весь файл в память. """ - inspections = [] - - # Декодируем и создаём поток - for encoding in ["utf-8", "windows-1251", "cp1251"]: - try: - xml_str = content.decode(encoding) - break - except UnicodeDecodeError: - continue - else: - xml_str = content.decode("utf-8", errors="replace") - - xml_str = self._sanitize_xml(xml_str) - - # Используем iterparse для потоковой обработки import io - xml_stream = io.StringIO(xml_str) - - # Определяем теги, которые нас интересуют + inspections = [] target_tags = {"INSPECTION", "inspection", "check", "КНМ"} - count = 0 - try: + + def parse_stream(xml_stream: io.BytesIO | io.StringIO) -> None: + nonlocal count for _event, elem in ET.iterparse(xml_stream, events=["end"]): # noqa: S314 - # Извлекаем имя тега без namespace tag_name = elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag + if tag_name not in target_tags: + continue + inspection = self._parse_xml_record(elem) + if inspection: + inspections.append(inspection) + count += 1 + if count % 10000 == 0: + logger.info("Streaming parsed %d inspections...", count) + elem.clear() - if tag_name in target_tags: - inspection = self._parse_xml_record(elem) - if inspection: - inspections.append(inspection) - count += 1 - - if count % 10000 == 0: - logger.info("Streaming parsed %d inspections...", count) - - # Очищаем элемент для освобождения памяти - elem.clear() - + try: + # ElementTree сам учитывает encoding из XML declaration. Для больших + # файлов это исключает несколько полноразмерных Unicode-копий XML. + parse_stream(io.BytesIO(content)) except ET.ParseError as e: - logger.error("XML streaming parse error at %d records: %s", count, e) if inspections: + logger.error("XML streaming parse error at %d records: %s", count, e) logger.info( "Returning %d successfully parsed records", len(inspections) ) + elif len(content) <= self.STREAMING_THRESHOLD_BYTES: + # Совместимость с небольшими файлами без корректной декларации + # кодировки и с историческим поведением очистки XML. + for encoding in ["utf-8", "windows-1251", "cp1251"]: + try: + xml_str = content.decode(encoding) + break + except UnicodeDecodeError: + continue + else: + xml_str = content.decode("utf-8", errors="replace") + try: + parse_stream(io.StringIO(self._sanitize_xml(xml_str))) + except ET.ParseError as fallback_error: + raise ProverkiClientError( + f"Failed to parse XML: {fallback_error}" + ) from fallback_error else: raise ProverkiClientError(f"Failed to parse XML: {e}") from e diff --git a/src/apps/parsers/frontend_compat.py b/src/apps/parsers/frontend_compat.py index 40fad2f..0f8b67a 100644 --- a/src/apps/parsers/frontend_compat.py +++ b/src/apps/parsers/frontend_compat.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from datetime import timedelta from typing import Any +from apps.core.models import JobStatus from apps.core.response import api_error_response, api_response from apps.core.services import BackgroundJobService from apps.parsers.models import ( @@ -46,7 +47,8 @@ SYSTEM_LOGS_TAG = "System Logs" ACTIVE_JOB_STATUSES = {"pending", "started", "retry"} SUCCESS_LOAD_STATUSES = {"success", "skipped"} ERROR_LOAD_STATUSES = {"failed", "failure", "error"} -STALE_ACTIVE_MAX_AGE_MINUTES = 90 +STALE_ACTIVE_MAX_AGE_MINUTES = 4 * 60 +STALE_PENDING_MAX_AGE_MINUTES = 24 * 60 PARSING_SETTINGS_CACHE_KEY = "parsers:frontend_compat:parsing_settings" PARSING_SETTINGS_FIELDS = { @@ -297,15 +299,34 @@ def _active_tasks_for_definition( for source_key in definition.source_keys if source_key in PARSER_SOURCES ] + now = timezone.now() + active_cutoff = now - timedelta( + minutes=int( + getattr( + settings, + "PARSER_STALE_LOAD_MAX_AGE_MINUTES", + STALE_ACTIVE_MAX_AGE_MINUTES, + ) + ) + ) + pending_cutoff = now - timedelta( + minutes=int( + getattr( + settings, + "PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES", + STALE_PENDING_MAX_AGE_MINUTES, + ) + ) + ) queryset = ( BackgroundJobService.get_queryset() + .filter(task_name__in=task_names) .filter( - task_name__in=task_names, - status__in=ACTIVE_JOB_STATUSES, - ) - .filter( - Q(started_at__isnull=False, started_at__gte=_stale_cutoff()) - | Q(started_at__isnull=True, created_at__gte=_stale_cutoff()) + Q(status=JobStatus.PENDING, created_at__gte=pending_cutoff) + | Q( + status__in=[JobStatus.STARTED, JobStatus.RETRY], + updated_at__gte=active_cutoff, + ) ) ) return [_serialize_active_job(job) for job in queryset.order_by("-created_at")[:10]] diff --git a/src/apps/parsers/migrations/0029_neutralize_external_collection_labels.py b/src/apps/parsers/migrations/0029_neutralize_external_collection_labels.py new file mode 100644 index 0000000..39c69b3 --- /dev/null +++ b/src/apps/parsers/migrations/0029_neutralize_external_collection_labels.py @@ -0,0 +1,18 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0028_registry_procurement_claims"), + ] + + operations = [ + migrations.AlterModelOptions( + name="checkocollectionattempt", + options={ + "ordering": ["-period_month", "source", "organization_id"], + "verbose_name": "попытка сбора внешних данных", + "verbose_name_plural": "попытки сбора внешних данных", + }, + ), + ] diff --git a/src/apps/parsers/migrations/0030_extend_stale_parser_job_timeouts.py b/src/apps/parsers/migrations/0030_extend_stale_parser_job_timeouts.py new file mode 100644 index 0000000..8431b27 --- /dev/null +++ b/src/apps/parsers/migrations/0030_extend_stale_parser_job_timeouts.py @@ -0,0 +1,39 @@ +import json + +from django.db import migrations + +CLEANUP_TASK_NAME = "parser:cleanup-stale-loads" +ACTIVE_MAX_AGE_MINUTES = 4 * 60 +PENDING_MAX_AGE_MINUTES = 24 * 60 + + +def extend_stale_parser_job_timeouts(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=CLEANUP_TASK_NAME).update( + kwargs=json.dumps( + { + "max_age_minutes": ACTIVE_MAX_AGE_MINUTES, + "pending_max_age_minutes": PENDING_MAX_AGE_MINUTES, + } + ) + ) + + +def restore_stale_parser_job_timeouts(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=CLEANUP_TASK_NAME).update( + kwargs=json.dumps({"max_age_minutes": 90}) + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0029_neutralize_external_collection_labels"), + ] + + operations = [ + migrations.RunPython( + extend_stale_parser_job_timeouts, + reverse_code=restore_stale_parser_job_timeouts, + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index b2763d1..161bacd 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -118,7 +118,7 @@ class ParserBatchSequence(TimestampMixin, models.Model): class CheckoCollectionAttempt(TimestampMixin, models.Model): - """Monthly Checko collection claim for one organization and source.""" + """Monthly external collection claim for one organization and source.""" class Source(models.TextChoices): ARBITRATION = "arbitration", _("Арбитражные дела") @@ -160,8 +160,8 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model): class Meta: db_table = "parsers_checko_collection_attempt" - verbose_name = _("попытка сбора Checko") - verbose_name_plural = _("попытки сбора Checko") + verbose_name = _("попытка сбора внешних данных") + verbose_name_plural = _("попытки сбора внешних данных") ordering = ["-period_month", "source", "organization_id"] constraints = [ models.UniqueConstraint( diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index 430a173..092c831 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -945,6 +945,7 @@ class SourceCardItemSerializer(serializers.Serializer): """Подисточник внутри агрегированной карточки.""" code = serializers.CharField(read_only=True) + refresh_key = serializers.CharField(read_only=True) title = serializers.CharField(read_only=True) description = serializers.CharField(read_only=True) parser_source = serializers.CharField(read_only=True, allow_null=True) diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index 88f5609..0434992 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -419,11 +419,14 @@ class ParserLoadLogService(BaseService[ParserLoadLog]): cls, *, max_age_minutes: int, + pending_max_age_minutes: int = 24 * 60, ) -> int: """Закрыть зависшие in_progress логи без живой свежей BackgroundJob.""" from apps.core.models import BackgroundJob, JobStatus - cutoff = timezone.now() - timedelta(minutes=max_age_minutes) + now = timezone.now() + cutoff = now - timedelta(minutes=max_age_minutes) + pending_cutoff = now - timedelta(minutes=pending_max_age_minutes) stale_logs = list( cls.model.objects.filter( status=ParserLoadLog.Status.IN_PROGRESS, @@ -442,8 +445,11 @@ class ParserLoadLogService(BaseService[ParserLoadLog]): meta__source=log.source, ).filter(Q(meta__batch_id=log.batch_id) | Q(meta__batch_id__isnull=True)) fresh_jobs = active_jobs.filter( - Q(started_at__isnull=False, started_at__gte=cutoff) - | Q(started_at__isnull=True, created_at__gte=cutoff) + Q(status=JobStatus.PENDING, created_at__gte=pending_cutoff) + | Q( + status__in=[JobStatus.STARTED, JobStatus.RETRY], + updated_at__gte=cutoff, + ) ) if fresh_jobs.exists(): continue @@ -451,8 +457,11 @@ class ParserLoadLogService(BaseService[ParserLoadLog]): cls.mark_failed(log, stale_message) updated += 1 stale_jobs = active_jobs.filter( - Q(started_at__isnull=False, started_at__lt=cutoff) - | Q(started_at__isnull=True, created_at__lt=cutoff) + Q(status=JobStatus.PENDING, created_at__lt=pending_cutoff) + | Q( + status__in=[JobStatus.STARTED, JobStatus.RETRY], + updated_at__lt=cutoff, + ) ) for job in stale_jobs.order_by("created_at"): job.fail(error=stale_message) diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index 29a7ac6..8fe2225 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -31,7 +31,8 @@ from rest_framework.exceptions import ValidationError SUCCESSFUL_LOAD_STATUSES = {"success", "skipped"} ACTIVE_JOB_STATUSES = [JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY] -STALE_ACTIVE_MAX_AGE_MINUTES = 90 +STALE_ACTIVE_MAX_AGE_MINUTES = 4 * 60 +STALE_PENDING_MAX_AGE_MINUTES = 24 * 60 SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60 @@ -55,6 +56,7 @@ class SourceItemDefinition: title: str description: str parser_source: str | None = None + refresh_key: str | None = None @dataclass(frozen=True) @@ -218,6 +220,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( "Реестр промышленной продукции, произведенной на территории РФ." ), parser_source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, + refresh_key="mpt_products", ), SourceItemDefinition( code="manufactures", @@ -769,17 +772,12 @@ class SourceCardService: active_tasks_by_slug: dict[str, list[dict[str, Any]]] = { slug: [] for slug in task_names_by_slug } - cutoff = cls._stale_cutoff() queryset = ( BackgroundJobService.get_queryset() .filter( task_name__in=list(slugs_by_task_name), - status__in=ACTIVE_JOB_STATUSES, - ) - .filter( - Q(started_at__isnull=False, started_at__gte=cutoff) - | Q(started_at__isnull=True, created_at__gte=cutoff) ) + .filter(cls._fresh_active_job_filter()) .order_by("-created_at") ) @@ -1157,6 +1155,7 @@ class SourceCardService: return { "code": item.code, + "refresh_key": item.refresh_key or item.code, "title": item.title, "description": item.description, "parser_source": item.parser_source, @@ -1288,17 +1287,12 @@ class SourceCardService: def _get_active_tasks( cls, definition: SourceCardDefinition ) -> list[dict[str, Any]]: - cutoff = cls._stale_cutoff() queryset = ( BackgroundJobService.get_queryset() .filter( task_name__in=definition.task_names, - status__in=ACTIVE_JOB_STATUSES, - ) - .filter( - Q(started_at__isnull=False, started_at__gte=cutoff) - | Q(started_at__isnull=True, created_at__gte=cutoff) ) + .filter(cls._fresh_active_job_filter()) ) return [ cls._serialize_job(job) for job in queryset.order_by("-created_at")[:10] @@ -1349,6 +1343,31 @@ class SourceCardService: ) return timezone.now() - timedelta(minutes=max_age_minutes) + @classmethod + def _fresh_active_job_filter(cls) -> Q: + now = timezone.now() + active_max_age_minutes = int( + getattr( + settings, + "PARSER_STALE_LOAD_MAX_AGE_MINUTES", + STALE_ACTIVE_MAX_AGE_MINUTES, + ) + ) + pending_max_age_minutes = int( + getattr( + settings, + "PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES", + STALE_PENDING_MAX_AGE_MINUTES, + ) + ) + return Q( + status=JobStatus.PENDING, + created_at__gte=now - timedelta(minutes=pending_max_age_minutes), + ) | Q( + status__in=[JobStatus.STARTED, JobStatus.RETRY], + updated_at__gte=now - timedelta(minutes=active_max_age_minutes), + ) + @classmethod def _is_stale_load(cls, latest_load: ParserLoadLog | None) -> bool: if latest_load is None or latest_load.status != "in_progress": diff --git a/src/apps/parsers/source_registry.py b/src/apps/parsers/source_registry.py index d9b14e3..880ea43 100644 --- a/src/apps/parsers/source_registry.py +++ b/src/apps/parsers/source_registry.py @@ -236,10 +236,10 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { status="implemented", upstream_url="https://kad.arbitr.ru/", access_method="official_search_api", - parser_strategy="checko_legal_cases_by_inn_ogrn", + parser_strategy="external_legal_cases_by_inn_ogrn", source_notes=( "Поиск дел выполняется по ИНН/ОГРН активных организаций из реестров. " - "Checko отдаёт карточки со ссылками на КАД Арбитр." + "Внешний сервис данных отдаёт карточки со ссылками на КАД Арбитр." ), api_route="arbitration/cases", ), @@ -258,7 +258,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { parser_strategy="fedresurs_bankruptcy_search", source_notes=( "Официальный ЕФРСБ; может отдавать anti-bot challenge worker'ам. " - "Если официальный портал недоступен, используется Checko API по " + "Если официальный портал недоступен, используется внешний сервис данных по " "организациям из реестров. " "Ручная загрузка разрешена только для выгрузок, переданных Сергеем." ), diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 2ad171a..21bada0 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -88,7 +88,8 @@ REGISTRY_INSPECTIONS_CHECKO_LIMIT = 1000 REGISTRY_CONTRACTS_CHECKO_LIMIT = 1000 REGISTRY_ENRICHMENT_BATCH_SIZE = 250 FSTEC_CHECKO_IDENTITY_LOOKUP_LIMIT = 1000 -PARSER_STALE_LOAD_MAX_AGE_MINUTES = 90 +PARSER_STALE_LOAD_MAX_AGE_MINUTES = 4 * 60 +PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES = 24 * 60 PARSER_SOFT_TIME_LIMIT_SECONDS = 15 * 60 PARSER_TIME_LIMIT_SECONDS = 20 * 60 INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS = 45 * 60 @@ -470,7 +471,7 @@ def _fetch_fedresurs_bankruptcy_records( file_path: str | None, proxies: list[str] | None, ) -> list[GenericParserItem]: - """Загрузить банкротства: официальный портал, затем fallback через Checko.""" + """Загрузить банкротства: официальный портал, затем внешний fallback.""" official_error: Exception | None = None try: official_records = _fetch_structured_records( @@ -482,14 +483,14 @@ def _fetch_fedresurs_bankruptcy_records( if official_records or file_url or file_path: return official_records logger.warning( - "Fedresurs official source returned no records, falling back to Checko" + "Fedresurs official source returned no records, using external fallback" ) except Exception as exc: if file_url or file_path: raise official_error = exc logger.warning( - "Fedresurs official source failed, falling back to Checko: %s", + "Fedresurs official source failed, using external fallback: %s", exc, ) records = _fetch_checko_bankruptcy_records(proxies=proxies) @@ -498,12 +499,12 @@ def _fetch_fedresurs_bankruptcy_records( if official_error is None: raise ParserSourceSkipped( "fedresurs official source returned no bankruptcy records; " - "Checko fallback returned no bankruptcy records" + "external fallback returned no bankruptcy records" ) if isinstance(official_error, HTTPClientError): raise ParserSourceSkipped( "fedresurs upstream is unavailable or blocked; " - "Checko fallback returned no bankruptcy records" + "external fallback returned no bankruptcy records" ) from official_error raise official_error @@ -580,7 +581,7 @@ def _enrich_fstec_record_identities( logger.info( "FSTEC identity enrichment completed: enriched=%d ambiguous=%d " - "local_candidates=%d checko_candidates=%d", + "local_candidates=%d external_candidates=%d", enriched_count, ambiguous_count, len(local_candidates), @@ -720,7 +721,7 @@ def _fstec_checko_identity_candidates( ) except CheckoError as exc: logger.info( - "Checko FSTEC identity lookup skipped for %s: %s", + "External FSTEC identity lookup skipped for %s: %s", applicant_name, exc, ) @@ -846,10 +847,10 @@ def _fetch_checko_bankruptcy_records( *, proxies: list[str] | None, ) -> list[GenericParserItem]: - """Получить ЕФРСБ-сообщения по организациям из наших реестров через Checko.""" + """Получить ЕФРСБ-сообщения по организациям через внешний сервис данных.""" api_key = getattr(settings, "CHECKO_API_KEY", "") if not api_key: - logger.warning("CHECKO_API_KEY is empty; Fedresurs fallback skipped") + logger.warning("External provider API key is empty; Fedresurs fallback skipped") return [] limit = _resolve_lookup_limit( @@ -861,7 +862,7 @@ def _fetch_checko_bankruptcy_records( default=FEDRESURS_CHECKO_FALLBACK_LIMIT, ) if limit <= 0: - logger.info("Fedresurs Checko fallback is disabled by limit=%s", limit) + logger.info("Fedresurs external fallback is disabled by limit=%s", limit) return [] targets = _active_registry_lookup_targets( limit=limit, @@ -893,7 +894,7 @@ def _fetch_checko_bankruptcy_records( except CheckoRateLimitError as exc: finish_collection(attempt, records_count=0, error=exc) logger.warning( - "Checko bankruptcy fallback stopped: quota/rate limit reached " + "External bankruptcy fallback stopped: quota/rate limit reached " "(status_code=%s)", exc.status_code, ) @@ -901,7 +902,7 @@ def _fetch_checko_bankruptcy_records( except CheckoError as exc: finish_collection(attempt, records_count=0, error=exc) logger.info( - "Checko bankruptcy lookup skipped for target=%s: %s", + "External bankruptcy lookup skipped for target=%s: %s", target.inn or target.ogrn, exc, ) @@ -920,7 +921,7 @@ def _fetch_checko_bankruptcy_records( attempt, records_count=len(records) - records_before, ) - logger.info("Fetched %d bankruptcy records through Checko fallback", len(records)) + logger.info("Fetched %d bankruptcy records through external fallback", len(records)) return records @@ -931,7 +932,7 @@ def _checko_bankruptcy_items( fallback_ogrn: str, fallback_name: str, ) -> list[GenericParserItem]: - """Преобразовать банкротные сообщения Checko в generic records.""" + """Преобразовать банкротные сообщения внешнего сервиса в generic records.""" inn = str(getattr(company, "inn", "") or fallback_inn) ogrn = str(getattr(company, "ogrn", "") or fallback_ogrn) name = getattr(company, "short_name", None) or fallback_name @@ -1084,14 +1085,16 @@ def _fetch_checko_arbitration_records( limit: int | None, proxies: list[str] | None, ) -> list[GenericParserItem]: - """Получить арбитражные дела по ИНН/ОГРН через Checko legal-cases API.""" + """Получить арбитражные дела по ИНН/ОГРН через внешний API.""" api_key = getattr(settings, "CHECKO_API_KEY", "") if not api_key: - raise ParserSourceSkipped("CHECKO_API_KEY is empty; arbitration parser skipped") + raise ParserSourceSkipped( + "External provider API key is empty; arbitration parser skipped" + ) resolved_limit = _resolve_arbitration_limit(limit) if resolved_limit <= 0: - logger.info("Arbitration Checko parser is disabled by limit=%s", limit) + logger.info("Arbitration external parser is disabled by limit=%s", limit) return [] subjects = _arbitration_subjects(resolved_limit) @@ -1128,7 +1131,7 @@ def _fetch_checko_arbitration_records( failed_lookups += 1 finish_collection(attempt, records_count=0, error=exc) logger.info( - "Checko arbitration lookup skipped for subject=%s: %s", + "External arbitration lookup skipped for subject=%s: %s", _arbitration_subject_key(subject), exc, ) @@ -1139,10 +1142,12 @@ def _fetch_checko_arbitration_records( ) if attempted_lookups and failed_lookups == attempted_lookups and not records: - raise ParserSourceSkipped("Checko arbitration lookups failed for all subjects") + raise ParserSourceSkipped( + "External arbitration lookups failed for all subjects" + ) logger.info( - "Fetched %d arbitration records through Checko for %d subjects", + "Fetched %d arbitration records through external service for %d subjects", len(records), len(subjects), ) @@ -1150,7 +1155,7 @@ def _fetch_checko_arbitration_records( def _checko_arbitration_item(case, *, subject: ArbitrationSubject) -> GenericParserItem: - """Преобразовать дело Checko в generic record.""" + """Преобразовать дело внешнего сервиса в generic record.""" case_number = getattr(case, "case_number", "") or "" filing_date = getattr(case, "filing_date", "") or "" role = _case_role_for_subject(case, subject) @@ -1335,11 +1340,11 @@ def _fetch_checko_registry_inspections( limit: int | None, proxies: list[str] | None, ) -> list[ProverkiInspection]: - """Получить проверки по активным организациям из реестров через Checko.""" + """Получить проверки по активным организациям через внешний сервис данных.""" api_key = getattr(settings, "CHECKO_API_KEY", "") if not api_key: raise ParserSourceSkipped( - "CHECKO_API_KEY is empty; registry inspections parser skipped" + "External provider API key is empty; registry inspections parser skipped" ) resolved_limit = _resolve_lookup_limit( @@ -1351,7 +1356,9 @@ def _fetch_checko_registry_inspections( ), ) if resolved_limit <= 0: - logger.info("Registry inspections Checko parser is disabled by limit=%s", limit) + logger.info( + "Registry inspections external parser is disabled by limit=%s", limit + ) return [] targets = _active_registry_lookup_targets( @@ -1391,7 +1398,7 @@ def _fetch_checko_registry_inspections( failed_lookups += 1 finish_collection(attempt, records_count=0, error=exc) logger.info( - "Checko inspections lookup skipped for target=%s: %s", + "External inspections lookup skipped for target=%s: %s", target.inn or target.ogrn, exc, ) @@ -1402,10 +1409,10 @@ def _fetch_checko_registry_inspections( ) if attempted_lookups and failed_lookups == attempted_lookups and not records: - raise ParserSourceSkipped("Checko inspections lookups failed for all targets") + raise ParserSourceSkipped("External inspections lookups failed for all targets") logger.info( - "Fetched %d inspections through Checko for %d registry organizations", + "Fetched %d inspections through external service for %d registry organizations", len(records), len(targets), ) @@ -1488,7 +1495,7 @@ def _checko_unfair_supplier_items( company, target: RegistryLookupTarget, ) -> list[GenericParserItem]: - """Convert Checko НедобПостЗап values into supplier-bound source records.""" + """Convert external НедобПостЗап values into supplier-bound source records.""" company_inn = _normalize_identifier(getattr(company, "inn", "")) company_ogrn = _normalize_identifier(getattr(company, "ogrn", "")) if target.inn and company_inn != target.inn: @@ -1563,10 +1570,12 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901 proxies: list[str] | None, organization_ids: list[str] | None = None, ) -> list[GenericParserItem]: - """Fetch RNP entries through one Checko /company request per OПК organization.""" + """Fetch RNP entries through one external request per OПК organization.""" api_key = getattr(settings, "CHECKO_API_KEY", "") if not api_key: - raise ParserSourceSkipped("CHECKO_API_KEY is empty; RNP parser skipped") + raise ParserSourceSkipped( + "External provider API key is empty; RNP parser skipped" + ) resolved_limit = _resolve_registry_enrichment_limit(limit) if resolved_limit <= 0: @@ -1604,13 +1613,13 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901 finish_collection(attempt, records_count=0, error=exc) failed_lookups += 1 rate_limited = True - logger.warning("Checko RNP lookup stopped: quota/rate limit reached") + logger.warning("External RNP lookup stopped: quota/rate limit reached") break except CheckoError as exc: finish_collection(attempt, records_count=0, error=exc) failed_lookups += 1 logger.info( - "Checko RNP lookup failed for target=%s: %s", + "External RNP lookup failed for target=%s: %s", target.inn or target.ogrn, exc, ) @@ -1628,7 +1637,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901 ) if not records and (rate_limited or failed_lookups == len(targets)): - raise ParserSourceSkipped("Checko RNP lookups failed for all targets") + raise ParserSourceSkipped("External RNP lookups failed for all targets") return records @@ -1729,11 +1738,11 @@ def _fetch_checko_registry_contract_records( limit: int | None, proxies: list[str] | None, ) -> list[GenericParserItem]: - """Получить контракты по активным организациям из реестров через Checko.""" + """Получить контракты по активным организациям через внешний сервис данных.""" api_key = getattr(settings, "CHECKO_API_KEY", "") if not api_key: raise ParserSourceSkipped( - "CHECKO_API_KEY is empty; registry contracts parser skipped" + "External provider API key is empty; registry contracts parser skipped" ) resolved_limit = _resolve_lookup_limit( @@ -1745,7 +1754,7 @@ def _fetch_checko_registry_contract_records( ), ) if resolved_limit <= 0: - logger.info("Registry contracts Checko parser is disabled by limit=%s", limit) + logger.info("Registry contracts external parser is disabled by limit=%s", limit) return [] targets = _active_registry_lookup_targets( @@ -1790,7 +1799,7 @@ def _fetch_checko_registry_contract_records( failed_lookups += 1 target_failures.append(exc) logger.info( - "Checko contracts lookup skipped for target=%s law=%s: %s", + "External contracts lookup skipped for target=%s law=%s: %s", target.inn or target.ogrn, law.value, exc, @@ -1803,10 +1812,10 @@ def _fetch_checko_registry_contract_records( expected_lookups = attempted_lookups * 2 if expected_lookups and failed_lookups == expected_lookups and not records: - raise ParserSourceSkipped("Checko contracts lookups failed for all targets") + raise ParserSourceSkipped("External contracts lookups failed for all targets") logger.info( - "Fetched %d contracts through Checko for %d registry organizations", + "Fetched %d contracts through external service for %d registry organizations", len(records), len(targets), ) @@ -3374,7 +3383,7 @@ def parse_unfair_suppliers( organization_ids: list[str] | None = None, requested_by_id: int | None = None, ) -> dict: - """Checko RNP lookup by default; explicit files remain a manual tool.""" + """External RNP lookup by default; explicit files remain a manual tool.""" proxies = _resolve_proxies(proxies) if file_url or file_path: @@ -3478,7 +3487,7 @@ def parse_registry_inspections( proxies: list[str] | None = None, requested_by_id: int | None = None, ) -> dict: - """Lookup проверок по активным организациям из реестров через Checko.""" + """Lookup проверок по активным организациям через внешний сервис данных.""" proxies = _resolve_proxies(proxies) return _run_inspection_parser( self, @@ -3596,7 +3605,10 @@ def parse_fstec_registers( @shared_task -def cleanup_stale_parser_loads(max_age_minutes: int | None = None) -> dict: +def cleanup_stale_parser_loads( + max_age_minutes: int | None = None, + pending_max_age_minutes: int | None = None, +) -> dict: """Закрыть stale in_progress загрузки и jobs после рестартов worker/deploy.""" if max_age_minutes is None: max_age_minutes = getattr( @@ -3604,14 +3616,22 @@ def cleanup_stale_parser_loads(max_age_minutes: int | None = None) -> dict: "PARSER_STALE_LOAD_MAX_AGE_MINUTES", PARSER_STALE_LOAD_MAX_AGE_MINUTES, ) + if pending_max_age_minutes is None: + pending_max_age_minutes = getattr( + settings, + "PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES", + PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES, + ) source_values = {descriptor.source for descriptor in PARSER_SOURCES.values()} task_names = {descriptor.task_name for descriptor in PARSER_SOURCES.values()} task_names.add("apps.parsers.tasks.scan_fns_directory") marked_failed = ParserLoadLogService.mark_stale_in_progress_failed( - max_age_minutes=int(max_age_minutes) + max_age_minutes=int(max_age_minutes), + pending_max_age_minutes=int(pending_max_age_minutes), ) marked_jobs_failed = BackgroundJobService.mark_stale_active_jobs_failed( max_age_minutes=int(max_age_minutes), + pending_max_age_minutes=int(pending_max_age_minutes), task_names=task_names, meta_sources=source_values, ) @@ -3620,6 +3640,7 @@ def cleanup_stale_parser_loads(max_age_minutes: int | None = None) -> dict: "marked_failed": marked_failed, "marked_jobs_failed": marked_jobs_failed, "max_age_minutes": int(max_age_minutes), + "pending_max_age_minutes": int(pending_max_age_minutes), } diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index e5ab40f..3702954 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -9,8 +9,10 @@ import csv import json import uuid from collections import defaultdict +from datetime import timedelta from apps.core.filters import BaseFilterSet +from apps.core.models import JobStatus from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from apps.core.response import api_error_response, api_response from apps.core.serializers import BackgroundJobListSerializer @@ -72,6 +74,7 @@ from django.core.paginator import Paginator from django.db.models import CharField, Count, Q from django.db.models.functions import Cast, Lower from django.http import HttpResponse +from django.utils import timezone from django.utils.text import get_valid_filename from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask from django_filters import rest_framework as filters @@ -151,6 +154,10 @@ TASKS_BY_NAME = { "apps.parsers.tasks.parse_trudvsem_vacancies": tasks.parse_trudvsem_vacancies, } +PARSER_SOURCE_ALIASES = { + "industrial_products": "mpt_products", +} + class MultipartFormSwaggerAutoSchema(SwaggerAutoSchema): """Document mixed JSON/multipart upload endpoints as form-data in Swagger.""" @@ -2642,21 +2649,52 @@ class ParserRunView(APIView): permission_classes = [IsAuthenticated] def post(self, request: Request, source_key: str): - descriptor = PARSER_SOURCES.get(source_key) + canonical_source_key = PARSER_SOURCE_ALIASES.get(source_key, source_key) + descriptor = PARSER_SOURCES.get(canonical_source_key) if descriptor is None: return _source_not_found_response(source_key) serializer = ParserRunRequestSerializer(data=request.data) serializer.is_valid(raise_exception=True) task = TASKS_BY_NAME[descriptor.task_name] task_kwargs = build_task_kwargs( - source_key, serializer.validated_data, request.user.id + canonical_source_key, serializer.validated_data, request.user.id ) + now = timezone.now() + active_job = ( + BackgroundJobService.get_queryset() + .filter(task_name=descriptor.task_name) + .filter( + Q( + status=JobStatus.PENDING, + created_at__gte=now - timedelta(hours=24), + ) + | Q( + status__in=[JobStatus.STARTED, JobStatus.RETRY], + updated_at__gte=now - timedelta(hours=4), + ) + ) + .order_by("-created_at") + .first() + ) + if active_job is not None: + return api_response( + { + "task_id": active_job.task_id, + "source": descriptor.source, + "task_name": descriptor.task_name, + "already_running": True, + }, + status_code=status.HTTP_202_ACCEPTED, + ) task_id = str(uuid.uuid4()) BackgroundJobService.create_job( task_id=task_id, task_name=descriptor.task_name, user_id=request.user.id, - meta={"source_key": source_key, "source": descriptor.source}, + meta={ + "source_key": canonical_source_key, + "source": descriptor.source, + }, ) async_result = task.apply_async(kwargs=task_kwargs, task_id=task_id) return api_response( diff --git a/src/organizations/management/commands/build_source_record_exports.py b/src/organizations/management/commands/build_source_record_exports.py new file mode 100644 index 0000000..6a7c8dc --- /dev/null +++ b/src/organizations/management/commands/build_source_record_exports.py @@ -0,0 +1,34 @@ +"""Build the complete prepared source-record export matrix.""" + +from __future__ import annotations + +import json + +from apps.core.management.commands.base import BaseAppCommand + +from organizations.source_record_export import build_source_record_export_artifacts + + +class Command(BaseAppCommand): + """Build source-record files synchronously for bootstrap and recovery.""" + + help = "Формирует готовые CSV/XLSX/JSON выгрузки источников" + use_transaction = False + + def execute_command(self, *args, **options) -> str: + generation = build_source_record_export_artifacts() + rendered = json.dumps( + { + "generation_id": generation.generation_id, + "generated_at": generation.generated_at, + "export_year": generation.export_year, + "artifacts_count": generation.artifacts_count, + "files_count": generation.files_count, + "records_count": generation.records_count, + "total_size": generation.total_size, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/organizations/management/commands/restore_fns_financial_lines.py b/src/organizations/management/commands/restore_fns_financial_lines.py new file mode 100644 index 0000000..7cd7f6d --- /dev/null +++ b/src/organizations/management/commands/restore_fns_financial_lines.py @@ -0,0 +1,186 @@ +"""Restore financial lines from already processed FNS workbooks.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from apps.core.management.commands.base import BaseAppCommand +from apps.parsers.clients.fns.parser import FNSExcelParser +from apps.parsers.models import ParserLoadLog +from django.conf import settings +from django.db import transaction +from django.db.models import Count + +from organizations.models import ( + OrganizationSourceFinancialLine, + OrganizationSourceRecord, +) + + +class Command(BaseAppCommand): + """Reparse successful FNS records whose normalized lines are missing.""" + + help = ( + "Восстанавливает отсутствующие строки финансовой отчетности из processed XLSX" + ) + use_transaction = False + + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument( + "--ogrn", + default=None, + help="Ограничить восстановление одной организацией по ОГРН/ОГРНИП.", + ) + parser.add_argument( + "--limit", + type=int, + default=None, + help="Обработать не более указанного количества отчетов.", + ) + + def execute_command(self, *args, **options) -> str: + records = self._records_to_restore( + ogrn=options.get("ogrn"), + limit=options.get("limit"), + ) + processed_directory = Path(settings.FNS_PROCESSED_DIRECTORY) + summary = { + "selected": len(records), + "parsed": 0, + "restored": 0, + "restored_lines": 0, + "skipped_concurrent": 0, + "missing_files": 0, + "failed": 0, + "dry_run": self.dry_run, + "errors": [], + } + + for record in self.progress_iter( + records, + desc="Восстановление финансовых строк", + total=len(records), + ): + try: + parsed = self._parse_record_file( + record=record, + processed_directory=processed_directory, + ) + except FileNotFoundError as exc: + summary["missing_files"] += 1 + summary["errors"].append( + {"external_id": record.external_id, "error": str(exc)} + ) + self.log_debug(str(exc)) + continue + except Exception as exc: # continue recovery and report exact total + summary["failed"] += 1 + summary["errors"].append( + {"external_id": record.external_id, "error": str(exc)} + ) + self.log_debug(f"{record.external_id}: {exc}") + continue + + summary["parsed"] += 1 + deduplicated_lines = { + (line.form_code, line.line_code, line.year): line + for line in parsed.lines + } + if self.dry_run: + summary["restored"] += 1 + summary["restored_lines"] += len(deduplicated_lines) + continue + + with transaction.atomic(): + locked_record = ( + OrganizationSourceRecord.objects.select_for_update().get( + pk=record.pk + ) + ) + if locked_record.financial_lines.exists(): + summary["skipped_concurrent"] += 1 + continue + + OrganizationSourceFinancialLine.objects.bulk_create( + [ + OrganizationSourceFinancialLine( + source_record=locked_record, + form_code=line.form_code, + line_code=line.line_code, + line_name=line.line_name, + year=line.year, + period_start=line.period_start, + period_end=line.period_end, + ) + for line in deduplicated_lines.values() + ], + batch_size=1000, + ) + payload = dict(locked_record.payload or {}) + payload["lines_count"] = len(parsed.lines) + locked_record.payload = payload + locked_record.save(update_fields=["payload", "updated_at"]) + + summary["restored"] += 1 + summary["restored_lines"] += len(deduplicated_lines) + + rendered = json.dumps(summary, ensure_ascii=False, sort_keys=True) + if summary["failed"] or summary["missing_files"]: + self.log_warning(rendered) + else: + self.log_success(rendered) + return rendered + + @staticmethod + def _records_to_restore( + *, + ogrn: str | None, + limit: int | None, + ) -> list[OrganizationSourceRecord]: + queryset = ( + OrganizationSourceRecord.objects.filter( + source=ParserLoadLog.Source.FNS_REPORTS, + ) + .select_related("extension", "extension__organization") + .annotate(financial_lines_count=Count("financial_lines")) + .filter(financial_lines_count=0) + .order_by("external_id", "uid") + ) + if ogrn: + queryset = queryset.filter( + extension__organization__ogrn=ogrn + ) | queryset.filter(extension__organization__ogrip=ogrn) + queryset = queryset.order_by("external_id", "uid") + if limit is not None: + if limit < 1: + raise ValueError("--limit должен быть положительным числом") + queryset = queryset[:limit] + return list(queryset) + + @staticmethod + def _parse_record_file( + *, + record: OrganizationSourceRecord, + processed_directory: Path, + ): + payload = dict(record.payload or {}) + file_name = str(payload.get("file_name") or record.title or "") + if not file_name or Path(file_name).name != file_name: + raise ValueError("Некорректное имя processed XLSX") + file_path = processed_directory / file_name + if not file_path.is_file(): + raise FileNotFoundError(f"Processed XLSX не найден: {file_name}") + + parsed = FNSExcelParser.parse_file(file_path) + if parsed.external_id != record.external_id: + raise ValueError("external_id файла не совпадает с source record") + expected_ogrn = str( + payload.get("ogrn") + or record.extension.organization.ogrn + or record.extension.organization.ogrip + ) + if parsed.ogrn != expected_ogrn: + raise ValueError("ОГРН файла не совпадает с source record") + return parsed diff --git a/src/organizations/migrations/0008_seed_nightly_source_record_exports.py b/src/organizations/migrations/0008_seed_nightly_source_record_exports.py new file mode 100644 index 0000000..b2b9002 --- /dev/null +++ b/src/organizations/migrations/0008_seed_nightly_source_record_exports.py @@ -0,0 +1,61 @@ +import json + +from django.db import migrations + +NIGHTLY_SOURCE_EXPORT_TASK_NAME = "organizations:source-record-exports:nightly-msk" +NIGHTLY_SOURCE_EXPORT_TASK_PATH = ( + "organizations.tasks.refresh_source_record_export_artifacts" +) +NIGHTLY_SOURCE_EXPORT_MSK_CRON = { + "minute": "30", + "hour": "5", + "day_of_week": "*", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "Europe/Moscow", +} + + +def seed_nightly_source_record_export_schedule(apps, schema_editor): + CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + crontab, _ = CrontabSchedule.objects.get_or_create(**NIGHTLY_SOURCE_EXPORT_MSK_CRON) + field_names = {field.name for field in PeriodicTask._meta.fields} + schedule_fields = {"crontab": crontab} + for field_name in ("interval", "solar", "clocked"): + if field_name in field_names: + schedule_fields[field_name] = None + + PeriodicTask.objects.update_or_create( + name=NIGHTLY_SOURCE_EXPORT_TASK_NAME, + defaults={ + "task": NIGHTLY_SOURCE_EXPORT_TASK_PATH, + "args": json.dumps([]), + "kwargs": json.dumps({}), + "enabled": True, + "description": ( + "Nightly preparation of source-record CSV/XLSX/JSON artifacts." + ), + **schedule_fields, + }, + ) + + +def remove_nightly_source_record_export_schedule(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=NIGHTLY_SOURCE_EXPORT_TASK_NAME).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("django_celery_beat", "0018_improve_crontab_helptext"), + ("organizations", "0007_auto_20260607_1017"), + ] + + operations = [ + migrations.RunPython( + seed_nightly_source_record_export_schedule, + reverse_code=remove_nightly_source_record_export_schedule, + ), + ] diff --git a/src/organizations/migrations/0009_source_record_export_year_indexes.py b/src/organizations/migrations/0009_source_record_export_year_indexes.py new file mode 100644 index 0000000..c721a2c --- /dev/null +++ b/src/organizations/migrations/0009_source_record_export_year_indexes.py @@ -0,0 +1,39 @@ +"""Add online indexes used by current-year source-record exports.""" + +from django.db import migrations + + +class Migration(migrations.Migration): + """Index normalized record years and dateless-record creation timestamps.""" + + atomic = False + + dependencies = [ + ("organizations", "0008_seed_nightly_source_record_exports"), + ] + + operations = [ + migrations.RunSQL( + sql=( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS " + "organizations_source_record_export_year_idx " + "ON organizations_source_record " + "(((substring(record_date FROM '([0-9]{4})'))::integer))" + ), + reverse_sql=( + "DROP INDEX CONCURRENTLY IF EXISTS " + "organizations_source_record_export_year_idx" + ), + ), + migrations.RunSQL( + sql=( + "CREATE INDEX CONCURRENTLY IF NOT EXISTS " + "organizations_source_record_created_at_idx " + "ON organizations_source_record (created_at)" + ), + reverse_sql=( + "DROP INDEX CONCURRENTLY IF EXISTS " + "organizations_source_record_created_at_idx" + ), + ), + ] diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index 1ca9309..8c30527 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -12,6 +12,30 @@ from organizations.models import ( ) from organizations.source_record_export import EXPORT_FORMATS +ARBITRATION_ROLE_LABELS = { + "plaintiff": "Истец", + "истец": "Истец", + "defendant": "Ответчик", + "ответчик": "Ответчик", + "third_party": "Третье лицо", + "третье_лицо": "Третье лицо", + "третья_сторона": "Третье лицо", +} + + +def _arbitration_role_label(payload: dict) -> str: + """Return the frontend-facing role label for an arbitration case.""" + target = payload.get("target") + if not isinstance(target, dict): + target = {} + + raw_role = payload.get("role") or target.get("role") or payload.get("party_role") + if not isinstance(raw_role, str): + return "" + + normalized_role = raw_role.strip().casefold().replace("-", "_").replace(" ", "_") + return ARBITRATION_ROLE_LABELS.get(normalized_role, "") + class OrganizationSourceFinancialLineSerializer(serializers.ModelSerializer): """Structured financial line under a source record.""" @@ -62,6 +86,7 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): many=True, read_only=True ) organization = serializers.SerializerMethodField() + payload = serializers.SerializerMethodField() source_group = serializers.CharField( source="extension.source_group", read_only=True ) @@ -98,6 +123,18 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): def get_record_date(self, obj) -> str | None: return getattr(obj, "canonical_record_date", obj.record_date) or None + @swagger_serializer_method( + serializer_or_field=serializers.JSONField(read_only=True), + ) + def get_payload(self, obj) -> dict | list | str | int | float | bool | None: + payload = obj.payload + if obj.extension.source_group != SourceGroup.ARBITRATION: + return payload + + response_payload = dict(payload) if isinstance(payload, dict) else {} + response_payload["role"] = _arbitration_role_label(response_payload) + return response_payload + @swagger_serializer_method( serializer_or_field=OrganizationSourceRecordOrganizationSerializer, ) @@ -155,6 +192,12 @@ class OrganizationSourceRecordExportRequestSerializer(serializers.Serializer): return value +class OrganizationSourceRecordExportDownloadSerializer(serializers.Serializer): + """One-time ticket submitted by a native browser download form.""" + + ticket = serializers.CharField(max_length=64, trim_whitespace=False) + + class OrganizationDirectoryImportUploadSerializer(serializers.Serializer): """Request for uploading the canonical organization directory XLSX.""" diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index 7701361..100491a 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -1,28 +1,63 @@ -"""ZIP export for organization source records.""" +"""Prepared file exports for organization source records.""" from __future__ import annotations import csv import json +import os +import re +import secrets +import shutil import zipfile -from collections.abc import Sequence +from collections.abc import Iterable, Iterator, Sequence from dataclasses import dataclass -from datetime import date, datetime +from datetime import UTC, date, datetime from decimal import Decimal -from io import BytesIO, StringIO -from typing import Any +from itertools import islice +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any, BinaryIO, cast +from uuid import uuid4 -from django.db.models import QuerySet +from django.conf import settings +from django.core.cache import cache +from django.db import connection +from django.db.models import Prefetch, Q, QuerySet, prefetch_related_objects +from django.db.models.expressions import RawSQL from django.utils import timezone from openpyxl import Workbook -from organizations.models import OrganizationSourceRecord, SourceGroup +from organizations.models import ( + OrganizationSourceFinancialLine, + OrganizationSourceRecord, + SourceGroup, +) EXPORT_FORMAT_CSV = "csv" EXPORT_FORMAT_XLSX = "xlsx" EXPORT_FORMAT_JSON = "json" EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON) FINANCIAL_SOURCE_GROUP = SourceGroup.FINANCIAL_INDICATORS.value +EXPORT_MANIFEST_VERSION = 2 +CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json" +GENERATION_MANIFEST_FILE_NAME = "manifest.json" +GENERATION_DIRECTORY_NAME = "generations" +SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE = 1000 +SOURCE_RECORD_EXPORT_ZIP_CHUNK_SIZE = 1024 * 1024 +EXCEL_MAX_DATA_ROWS_PER_SHEET = 1_048_575 +DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000 +DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60 +SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket" +SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}") +EXPORT_PROVIDER_URL_PATTERN = re.compile( + r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*", + flags=re.IGNORECASE, +) +EXPORT_PROVIDER_NAME_PATTERN = re.compile( + r"(? int: + return len( + { + (artifact.source_group, artifact.file_format) + for artifact in self.artifacts + } + ) + + @property + def files_count(self) -> int: + return len(self.artifacts) + + @property + def total_size(self) -> int: + return sum(artifact.size for artifact in self.artifacts) + + @dataclass(frozen=True) class SourceRecordExportArchive: - """In-memory source records export archive.""" + """A request-specific ZIP streamed only from prepared files.""" archive_name: str - archive_bytes: bytes + archive_chunks: Iterable[bytes] files_count: int + generated_at: str + + +@dataclass(frozen=True) +class SourceRecordExportDownloadTicket: + """Short-lived capability for one native browser download.""" + + ticket: str + archive_name: str + expires_in: int + + +class _StreamingZipSink: + """Unseekable zipfile target whose written chunks can be drained.""" + + def __init__(self) -> None: + self._offset = 0 + self._chunks: list[bytes] = [] + + def write(self, data: bytes) -> int: + rendered_data = bytes(data) + self._chunks.append(rendered_data) + self._offset += len(rendered_data) + return len(rendered_data) + + def tell(self) -> int: + return self._offset + + def flush(self) -> None: + return None + + def drain(self) -> tuple[bytes, ...]: + chunks = tuple(self._chunks) + self._chunks.clear() + return chunks + + +def build_source_record_export_artifacts( + *, + now: datetime | None = None, + export_directory: str | Path | None = None, +) -> SourceRecordExportGeneration: + """Build all files on disk and atomically publish a new generation.""" + + root_directory = _resolve_export_directory(export_directory) + generations_directory = root_directory / GENERATION_DIRECTORY_NAME + root_directory.mkdir(parents=True, exist_ok=True) + generations_directory.mkdir(parents=True, exist_ok=True) + + generated_at_datetime = _normalize_generation_datetime(now or timezone.now()) + export_year = _export_year(generated_at_datetime) + generation_id = ( + f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}" + ) + staging_directory = generations_directory / f".building-{generation_id}" + final_directory = generations_directory / generation_id + staging_directory.mkdir() + + try: + artifacts: list[SourceRecordExportArtifact] = [] + source_record_counts: dict[str, int] = {} + + for source_group in SOURCE_GROUP_EXPORT_FILE_STEMS: + row_spool_path = staging_directory / f".{source_group}.rows.json" + headers, records_count = _spool_source_group_rows( + source_group=source_group, + output_path=row_spool_path, + export_year=export_year, + ) + source_record_counts[source_group] = records_count + + try: + for file_format in _source_group_export_formats(source_group): + file_name = _build_source_group_file_name( + source_group=source_group, + file_format=file_format, + ) + artifact_paths = _render_source_group_artifact( + row_spool_path=row_spool_path, + output_path=staging_directory / file_name, + headers=headers, + file_format=file_format, + records_count=records_count, + ) + parts_count = len(artifact_paths) + artifacts.extend( + SourceRecordExportArtifact( + source_group=source_group, + file_format=file_format, + file_name=artifact_path.name, + path=final_directory / artifact_path.name, + size=artifact_path.stat().st_size, + records_count=records_count, + part_number=part_number, + parts_count=parts_count, + ) + for part_number, artifact_path in enumerate( + artifact_paths, + start=1, + ) + ) + finally: + row_spool_path.unlink(missing_ok=True) + + generation = SourceRecordExportGeneration( + generation_id=generation_id, + generated_at=generated_at_datetime.isoformat(), + export_year=export_year, + artifacts=tuple(artifacts), + records_count=sum(source_record_counts.values()), + ) + manifest_payload = _generation_manifest_payload( + generation, + root_directory=root_directory, + ) + _write_json_file( + staging_directory / GENERATION_MANIFEST_FILE_NAME, + manifest_payload, + ) + os.replace(staging_directory, final_directory) + _write_json_file_atomically( + root_directory / CURRENT_EXPORT_MANIFEST_FILE_NAME, + manifest_payload, + ) + _cleanup_stale_generations( + generations_directory=generations_directory, + current_generation_id=generation_id, + ) + return generation + except Exception: + if staging_directory.exists(): + shutil.rmtree(staging_directory) + raise + + +def load_current_source_record_export_generation( + *, + export_directory: str | Path | None = None, +) -> SourceRecordExportGeneration: + """Load and validate the atomically published current generation.""" + + root_directory = _resolve_export_directory(export_directory) + manifest_path = root_directory / CURRENT_EXPORT_MANIFEST_FILE_NAME + try: + manifest_payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError) as exc: + raise SourceRecordExportArtifactsUnavailable( + "Prepared source-record export is not available." + ) from exc + + return _generation_from_manifest( + manifest_payload, + root_directory=root_directory, + ) def build_source_records_export_archive( *, source_groups: Sequence[str], export_format: str, - now: datetime | None = None, + export_directory: str | Path | None = None, + requested_at: datetime | None = None, ) -> SourceRecordExportArchive: - """Build a ZIP archive with one source-record file per source group.""" + """Package selected prepared files without querying source-record tables.""" - timestamp = (now or timezone.now()).strftime("%Y%m%d_%H%M%S") - archive_buffer = BytesIO() + root_directory = _resolve_export_directory(export_directory) + generation = load_current_source_record_export_generation( + export_directory=root_directory, + ) + requested_at_datetime = _normalize_generation_datetime( + requested_at or timezone.now() + ) + if generation.export_year != _export_year(requested_at_datetime): + raise SourceRecordExportArtifactsUnavailable( + "Prepared source-record export belongs to a different calendar year." + ) + artifacts_by_key: dict[ + tuple[str, str], + list[SourceRecordExportArtifact], + ] = {} + for artifact in generation.artifacts: + artifacts_by_key.setdefault( + (artifact.source_group, artifact.file_format), + [], + ).append(artifact) + selected_artifacts: list[SourceRecordExportArtifact] = [] - with zipfile.ZipFile( - archive_buffer, - mode="w", - compression=zipfile.ZIP_DEFLATED, - ) as archive: - for source_group in source_groups: - file_format = _resolve_source_group_export_format( - source_group=source_group, - requested_format=export_format, - ) - file_name = _build_source_group_file_name( - source_group=source_group, - file_format=file_format, - ) - archive.writestr( - file_name, - _render_source_group_file( - source_group=source_group, - file_format=file_format, - ), + for source_group in source_groups: + file_format = _resolve_source_group_export_format( + source_group=source_group, + requested_format=export_format, + ) + artifacts = artifacts_by_key.get((source_group, file_format), []) + if not artifacts or any(not artifact.path.is_file() for artifact in artifacts): + raise SourceRecordExportArtifactsUnavailable( + f"Prepared export artifact is missing: {source_group}/{file_format}." ) + selected_artifacts.extend( + sorted(artifacts, key=lambda artifact: artifact.part_number) + ) return SourceRecordExportArchive( - archive_name=f"organization_source_records_export_{timestamp}.zip", - archive_bytes=archive_buffer.getvalue(), - files_count=len(source_groups), + archive_name=_build_source_records_archive_name( + source_groups=source_groups, + requested_at=requested_at_datetime, + ), + archive_chunks=_stream_zip_archive(selected_artifacts), + files_count=len(selected_artifacts), + generated_at=generation.generated_at, ) +def create_source_record_export_download_ticket( + *, + source_groups: Sequence[str], + export_format: str, +) -> SourceRecordExportDownloadTicket: + """Validate prepared files and cache a short-lived download capability.""" + + requested_at = _normalize_generation_datetime(timezone.now()) + package = build_source_records_export_archive( + source_groups=source_groups, + export_format=export_format, + requested_at=requested_at, + ) + expires_in = max( + 1, + int( + getattr( + settings, + "SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS", + DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS, + ) + ), + ) + payload = { + "sources": list(source_groups), + "format": export_format, + "requested_at": requested_at.isoformat(), + } + for _attempt in range(3): + ticket = secrets.token_urlsafe(32) + if cache.add( + _source_record_export_ticket_cache_key(ticket), + payload, + timeout=expires_in, + ): + return SourceRecordExportDownloadTicket( + ticket=ticket, + archive_name=package.archive_name, + expires_in=expires_in, + ) + raise RuntimeError("Could not allocate a source-record export download ticket.") + + +def consume_source_record_export_download_ticket( + ticket: str, +) -> SourceRecordExportArchive: + """Consume a download ticket before streaming the prepared archive.""" + + if not SOURCE_RECORD_EXPORT_TICKET_PATTERN.fullmatch(ticket): + raise SourceRecordExportTicketInvalid + + cache_key = _source_record_export_ticket_cache_key(ticket) + payload = cache.get(cache_key) + if payload is None: + raise SourceRecordExportTicketInvalid + cache.delete(cache_key) + + try: + source_groups = payload["sources"] + export_format = payload["format"] + requested_at_value = payload["requested_at"] + if ( + not isinstance(source_groups, list) + or not source_groups + or any( + not isinstance(source_group, str) + or source_group not in SOURCE_GROUP_EXPORT_FILE_STEMS + for source_group in source_groups + ) + or len(source_groups) != len(set(source_groups)) + or export_format not in EXPORT_FORMATS + or not isinstance(requested_at_value, str) + ): + raise ValueError + requested_at = datetime.fromisoformat(requested_at_value) + if timezone.is_naive(requested_at): + raise ValueError + except (KeyError, TypeError, ValueError): + raise SourceRecordExportTicketInvalid from None + + return build_source_records_export_archive( + source_groups=source_groups, + export_format=export_format, + requested_at=requested_at, + ) + + +def _source_record_export_ticket_cache_key(ticket: str) -> str: + return f"{SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX}:{ticket}" + + +def _stream_zip_archive( + artifacts: Sequence[SourceRecordExportArtifact], +) -> Iterator[bytes]: + sink = _StreamingZipSink() + with zipfile.ZipFile( + cast(BinaryIO, sink), + mode="w", + compression=zipfile.ZIP_STORED, + allowZip64=True, + ) as archive: + for artifact in artifacts: + with artifact.path.open("rb") as source_file: + with archive.open( + artifact.file_name, + mode="w", + force_zip64=True, + ) as archive_entry: + yield from sink.drain() + while chunk := source_file.read( + SOURCE_RECORD_EXPORT_ZIP_CHUNK_SIZE + ): + archive_entry.write(chunk) + yield from sink.drain() + yield from sink.drain() + yield from sink.drain() + + +def _resolve_export_directory(export_directory: str | Path | None) -> Path: + if export_directory is not None: + return Path(export_directory) + + configured_directory = getattr( + settings, + "SOURCE_RECORD_EXPORT_DIRECTORY", + Path(settings.MEDIA_ROOT) / "source-record-exports", + ) + return Path(str(configured_directory)) + + +def _normalize_generation_datetime(value: datetime) -> datetime: + if timezone.is_naive(value): + value = timezone.make_aware(value, UTC) + return value.astimezone(UTC) + + +def _export_year(value: datetime) -> int: + """Return the calendar year in the configured service timezone.""" + return timezone.localtime(value).year + + +def _build_source_records_archive_name( + *, + source_groups: Sequence[str], + requested_at: datetime, +) -> str: + source_name = "__".join( + SOURCE_GROUP_EXPORT_FILE_STEMS[source_group] for source_group in source_groups + ) + timestamp = timezone.localtime(requested_at).strftime("%Y%m%d_%H%M%S") + return f"{source_name}_{timestamp}.zip" + + +def _source_group_export_formats(source_group: str) -> tuple[str, ...]: + if source_group == FINANCIAL_SOURCE_GROUP: + return (EXPORT_FORMAT_JSON,) + return EXPORT_FORMATS + + def _resolve_source_group_export_format( *, source_group: str, @@ -126,132 +540,245 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str return f"{SOURCE_GROUP_EXPORT_FILE_STEMS[source_group]}.{file_format}" -def _source_group_queryset(source_group: str) -> QuerySet[OrganizationSourceRecord]: - return ( +def _source_group_queryset( + source_group: str, + *, + export_year: int, +) -> QuerySet[OrganizationSourceRecord]: + queryset = ( OrganizationSourceRecord.objects.filter(extension__source_group=source_group) .select_related("extension", "extension__organization") - .prefetch_related("financial_lines") - .order_by( - "extension__organization__name", - "source", - "record_date", - "uid", + # Export order is not part of the file contract. Clearing the model's + # default ordering avoids a multi-gigabyte PostgreSQL disk sort for + # source groups with millions of rows. + .order_by() + ) + if source_group == FINANCIAL_SOURCE_GROUP: + return queryset.filter(financial_lines__year=export_year).distinct() + + if connection.vendor == "postgresql": + queryset = queryset.annotate( + export_record_year=RawSQL( + "substring(record_date FROM '([0-9]{4})')::integer", + (), + ) ) + return queryset.filter( + Q(export_record_year=export_year) + | Q(record_date="", created_at__year=export_year) + ) + + year_pattern = rf"(^|[^0-9]){export_year}([^0-9]|$)" + return queryset.filter( + Q(record_date__regex=year_pattern) + | Q(record_date="", created_at__year=export_year) ) -def _render_source_group_file(*, source_group: str, file_format: str) -> bytes: - queryset = _source_group_queryset(source_group) - payload_headers = _collect_payload_headers(queryset) - headers = [ +def _iter_source_records( + *, + source_group: str, + include_financial_lines: bool, + export_year: int, +) -> Iterator[OrganizationSourceRecord]: + iterator = _source_group_queryset( + source_group, + export_year=export_year, + ).iterator(chunk_size=SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE) + while True: + batch = list(islice(iterator, SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE)) + if not batch: + return + if include_financial_lines: + prefetch_related_objects( + batch, + Prefetch( + "financial_lines", + queryset=OrganizationSourceFinancialLine.objects.filter( + year=export_year + ).order_by(), + ), + ) + yield from batch + + +def _spool_source_group_rows( + *, + source_group: str, + output_path: Path, + export_year: int, +) -> tuple[list[str], int]: + include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP + payload_headers: set[str] = set() + records_count = 0 + + with output_path.open("w", encoding="utf-8", newline="") as output: + output.write("[") + is_first_row = True + for record in _iter_source_records( + source_group=source_group, + include_financial_lines=include_financial_lines, + export_year=export_year, + ): + row = _build_record_row( + record, + include_financial_lines=include_financial_lines, + ) + payload_headers.update(key for key in row if key.startswith("payload.")) + if is_first_row: + output.write("\n") + is_first_row = False + else: + output.write(",\n") + output.write(json.dumps(row, ensure_ascii=False, separators=(",", ":"))) + records_count += 1 + if not is_first_row: + output.write("\n") + output.write("]") + + return [ *ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS, - *payload_headers, - ] - include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP + *sorted(payload_headers), + ], records_count + +def _render_source_group_artifact( + *, + row_spool_path: Path, + output_path: Path, + headers: Sequence[str], + file_format: str, + records_count: int, +) -> tuple[Path, ...]: if file_format == EXPORT_FORMAT_CSV: - return _render_csv_file( - queryset=_source_group_queryset(source_group), + _render_csv_file( + row_spool_path=row_spool_path, + output_path=output_path, headers=headers, - payload_headers=payload_headers, ) + return (output_path,) if file_format == EXPORT_FORMAT_XLSX: - return _render_xlsx_file( - queryset=_source_group_queryset(source_group), + return _render_xlsx_files( + row_spool_path=row_spool_path, + output_path=output_path, headers=headers, - payload_headers=payload_headers, + records_count=records_count, ) - return _render_json_file( - queryset=_source_group_queryset(source_group), - headers=headers, - payload_headers=payload_headers, - include_financial_lines=include_financial_lines, - ) + _render_json_file(row_spool_path=row_spool_path, output_path=output_path) + return (output_path,) -def _collect_payload_headers( - queryset: QuerySet[OrganizationSourceRecord], -) -> list[str]: - payload_headers: set[str] = set() - - for record in queryset.iterator(chunk_size=1000): - payload_headers.update(_flatten_payload(record.payload).keys()) - - return sorted(payload_headers) +def _iter_spooled_rows(row_spool_path: Path) -> Iterator[dict[str, Any]]: + with row_spool_path.open("r", encoding="utf-8") as rows_file: + for line in rows_file: + serialized_row = line.strip() + if not serialized_row or serialized_row in {"[", "]", "[]"}: + continue + if serialized_row.endswith(","): + serialized_row = serialized_row[:-1] + yield json.loads(serialized_row) def _render_csv_file( *, - queryset: QuerySet[OrganizationSourceRecord], + row_spool_path: Path, + output_path: Path, headers: Sequence[str], - payload_headers: Sequence[str], -) -> bytes: - output = StringIO() - writer = csv.DictWriter(output, fieldnames=list(headers), lineterminator="\n") - writer.writeheader() - - for record in queryset.iterator(chunk_size=1000): - row = _build_record_row(record, payload_headers=payload_headers) - writer.writerow({key: _serialize_flat_value(row.get(key)) for key in headers}) - - return ("\ufeff" + output.getvalue()).encode("utf-8") +) -> None: + with output_path.open("w", encoding="utf-8-sig", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(headers), lineterminator="\n") + writer.writeheader() + for row in _iter_spooled_rows(row_spool_path): + writer.writerow( + {key: _serialize_flat_value(row.get(key)) for key in headers} + ) -def _render_xlsx_file( +def _render_xlsx_files( *, - queryset: QuerySet[OrganizationSourceRecord], + row_spool_path: Path, + output_path: Path, headers: Sequence[str], - payload_headers: Sequence[str], -) -> bytes: + records_count: int, +) -> tuple[Path, ...]: + rows_per_file = min( + EXCEL_MAX_DATA_ROWS_PER_SHEET, + max( + 1, + int( + getattr( + settings, + "SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE", + DEFAULT_XLSX_DATA_ROWS_PER_FILE, + ) + ), + ), + ) + parts_count = max(1, (records_count + rows_per_file - 1) // rows_per_file) + part_paths = tuple( + _build_xlsx_part_path( + output_path=output_path, + part_number=part_number, + parts_count=parts_count, + ) + for part_number in range(1, parts_count + 1) + ) + part_number = 1 + rows_in_file = 0 + workbook, worksheet = _new_export_workbook(headers) + + for row in _iter_spooled_rows(row_spool_path): + if rows_in_file >= rows_per_file: + workbook.save(part_paths[part_number - 1]) + workbook.close() + part_number += 1 + rows_in_file = 0 + workbook, worksheet = _new_export_workbook(headers) + worksheet.append([_serialize_flat_value(row.get(key)) for key in headers]) + rows_in_file += 1 + + workbook.save(part_paths[part_number - 1]) + workbook.close() + if part_number != parts_count: + raise ValueError("Unexpected XLSX source-record export parts count.") + return part_paths + + +def _new_export_workbook(headers: Sequence[str]): workbook = Workbook(write_only=True) worksheet = workbook.create_sheet(title="data") worksheet.append(list(headers)) + return workbook, worksheet - for record in queryset.iterator(chunk_size=1000): - row = _build_record_row(record, payload_headers=payload_headers) - worksheet.append([_serialize_flat_value(row.get(key)) for key in headers]) - output = BytesIO() - workbook.save(output) - return output.getvalue() +def _build_xlsx_part_path( + *, + output_path: Path, + part_number: int, + parts_count: int, +) -> Path: + if parts_count == 1: + return output_path + return output_path.with_name( + f"{output_path.stem}-part-{part_number:03d}{output_path.suffix}" + ) def _render_json_file( *, - queryset: QuerySet[OrganizationSourceRecord], - headers: Sequence[str], - payload_headers: Sequence[str], - include_financial_lines: bool, -) -> bytes: - rows = [] - - for record in queryset.iterator(chunk_size=1000): - row = _build_record_row(record, payload_headers=payload_headers) - json_row = {key: _serialize_json_value(row.get(key)) for key in headers} - if include_financial_lines: - json_row["financial_lines"] = [ - { - field_name: _serialize_json_value( - getattr(financial_line, field_name) - ) - for field_name in FINANCIAL_LINE_FIELDS - } - for financial_line in record.financial_lines.all() - ] - rows.append(json_row) - - return json.dumps(rows, ensure_ascii=False, indent=2).encode("utf-8") + row_spool_path: Path, + output_path: Path, +) -> None: + os.link(row_spool_path, output_path) def _build_record_row( record: OrganizationSourceRecord, *, - payload_headers: Sequence[str], + include_financial_lines: bool, ) -> dict[str, Any]: organization = record.extension.organization - payload = _flatten_payload(record.payload) - row: dict[str, Any] = { "Наименование": organization.full_name or organization.short_name @@ -259,6 +786,7 @@ def _build_record_row( "ИНН": organization.inn, "ОГРН": organization.ogrn, "КПП": organization.kpp, + "ОКПО": organization.okpo, "uid": str(record.uid), "source_group": record.extension.source_group, "source": record.source, @@ -272,12 +800,37 @@ def _build_record_row( "load_batch": record.load_batch, "created_at": record.created_at, "updated_at": record.updated_at, + **_flatten_payload(record.payload), + } + public_row = _sanitize_export_provider_value(row) + serialized_row = { + key: _serialize_json_value(value) for key, value in public_row.items() } - for header in payload_headers: - row[header] = payload.get(header) + if include_financial_lines: + serialized_row["financial_lines"] = [ + { + field_name: _serialize_json_value(getattr(financial_line, field_name)) + for field_name in FINANCIAL_LINE_FIELDS + } + for financial_line in record.financial_lines.all() + ] + return serialized_row - return row + +def _sanitize_export_provider_value(value: Any) -> Any: + """Remove provider wording from files without changing stored source data.""" + if isinstance(value, str): + sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value) + return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized) + if isinstance(value, dict): + return { + _sanitize_export_provider_value(key): _sanitize_export_provider_value(item) + for key, item in value.items() + } + if isinstance(value, list | tuple): + return [_sanitize_export_provider_value(item) for item in value] + return value def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]: @@ -303,7 +856,9 @@ def _serialize_flat_value(value: Any) -> str | int | float | bool: return str(value) if isinstance(value, datetime | date): return value.isoformat() - return value + if isinstance(value, str | int | float | bool): + return value + return str(value) def _serialize_json_value(value: Any) -> Any: @@ -314,3 +869,196 @@ def _serialize_json_value(value: Any) -> Any: if isinstance(value, datetime | date): return value.isoformat() return value + + +def _generation_manifest_payload( + generation: SourceRecordExportGeneration, + *, + root_directory: Path, +) -> dict[str, Any]: + return { + "version": EXPORT_MANIFEST_VERSION, + "generation_id": generation.generation_id, + "generated_at": generation.generated_at, + "export_year": generation.export_year, + "records_count": generation.records_count, + "artifacts_count": generation.artifacts_count, + "files_count": generation.files_count, + "total_size": generation.total_size, + "artifacts": [ + { + "source_group": artifact.source_group, + "format": artifact.file_format, + "file_name": artifact.file_name, + "relative_path": str(artifact.path.relative_to(root_directory)), + "size": artifact.size, + "records_count": artifact.records_count, + "part_number": artifact.part_number, + "parts_count": artifact.parts_count, + } + for artifact in generation.artifacts + ], + } + + +def _generation_from_manifest( + payload: Any, + *, + root_directory: Path, +) -> SourceRecordExportGeneration: + try: + if payload["version"] != EXPORT_MANIFEST_VERSION: + raise ValueError("Unsupported source-record export manifest version.") + generation_id = str(payload["generation_id"]) + generated_at = str(payload["generated_at"]) + datetime.fromisoformat(generated_at) + export_year = int(payload["export_year"]) + if not 1 <= export_year <= 9999: + raise ValueError("Source-record export year is invalid.") + records_count = int(payload["records_count"]) + artifact_payloads = payload["artifacts"] + if not isinstance(artifact_payloads, list): + raise TypeError("Manifest artifacts must be a list.") + + root_resolved = root_directory.resolve() + artifacts: list[SourceRecordExportArtifact] = [] + for artifact_payload in artifact_payloads: + artifact_path = root_directory / str(artifact_payload["relative_path"]) + artifact_path.resolve().relative_to(root_resolved) + if not artifact_path.is_file(): + raise FileNotFoundError(artifact_path) + source_group = str(artifact_payload["source_group"]) + file_format = str(artifact_payload["format"]) + file_name = str(artifact_payload["file_name"]) + artifact_size = int(artifact_payload["size"]) + part_number = int(artifact_payload.get("part_number", 1)) + parts_count = int(artifact_payload.get("parts_count", 1)) + expected_file_name = _build_source_group_file_name( + source_group=source_group, + file_format=file_format, + ) + if file_format == EXPORT_FORMAT_XLSX: + expected_file_name = _build_xlsx_part_path( + output_path=Path(expected_file_name), + part_number=part_number, + parts_count=parts_count, + ).name + if file_name != expected_file_name: + raise ValueError("Unexpected source-record export artifact name.") + if artifact_size != artifact_path.stat().st_size: + raise ValueError("Source-record export artifact size mismatch.") + artifacts.append( + SourceRecordExportArtifact( + source_group=source_group, + file_format=file_format, + file_name=file_name, + path=artifact_path, + size=artifact_size, + records_count=int(artifact_payload["records_count"]), + part_number=part_number, + parts_count=parts_count, + ) + ) + + _validate_manifest_artifact_parts(artifacts) + except (KeyError, TypeError, ValueError, OSError) as exc: + raise SourceRecordExportArtifactsUnavailable( + "Prepared source-record export manifest is invalid." + ) from exc + + return SourceRecordExportGeneration( + generation_id=generation_id, + generated_at=generated_at, + export_year=export_year, + artifacts=tuple(artifacts), + records_count=records_count, + ) + + +def _validate_manifest_artifact_parts( + artifacts: Sequence[SourceRecordExportArtifact], +) -> None: + expected_artifact_keys = { + (source_group, file_format) + for source_group in SOURCE_GROUP_EXPORT_FILE_STEMS + for file_format in _source_group_export_formats(source_group) + } + artifacts_by_key: dict[ + tuple[str, str], + list[SourceRecordExportArtifact], + ] = {} + for artifact in artifacts: + artifacts_by_key.setdefault( + (artifact.source_group, artifact.file_format), + [], + ).append(artifact) + if set(artifacts_by_key) != expected_artifact_keys: + raise ValueError("Source-record export manifest matrix is incomplete.") + + for artifact_key, artifact_parts in artifacts_by_key.items(): + parts_count = len(artifact_parts) + if ( + {artifact.parts_count for artifact in artifact_parts} != {parts_count} + or {artifact.part_number for artifact in artifact_parts} + != set(range(1, parts_count + 1)) + or (artifact_key[1] != EXPORT_FORMAT_XLSX and parts_count != 1) + ): + raise ValueError("Source-record export artifact parts are invalid.") + + +def _write_json_file(file_path: Path, payload: dict[str, Any]) -> None: + file_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def _write_json_file_atomically(file_path: Path, payload: dict[str, Any]) -> None: + temp_path: Path | None = None + try: + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=file_path.parent, + prefix=f".{file_path.name}.", + suffix=".tmp", + delete=False, + ) as temp_file: + json.dump(payload, temp_file, ensure_ascii=False, indent=2) + temp_file.flush() + os.fsync(temp_file.fileno()) + temp_path = Path(temp_file.name) + os.replace(temp_path, file_path) + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) + + +def _cleanup_stale_generations( + *, + generations_directory: Path, + current_generation_id: str, +) -> None: + generations_to_keep = max( + 1, + int(getattr(settings, "SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP", 2)), + ) + published_generations = sorted( + ( + path + for path in generations_directory.iterdir() + if path.is_dir() + and not path.name.startswith(".") + and (path / GENERATION_MANIFEST_FILE_NAME).is_file() + ), + key=lambda path: path.name, + reverse=True, + ) + retained_names = {current_generation_id} + retained_names.update( + path.name for path in published_generations[:generations_to_keep] + ) + + for generation_directory in published_generations: + if generation_directory.name not in retained_names: + shutil.rmtree(generation_directory) diff --git a/src/organizations/tasks.py b/src/organizations/tasks.py index 4d50d8a..73efa83 100644 --- a/src/organizations/tasks.py +++ b/src/organizations/tasks.py @@ -5,14 +5,51 @@ from __future__ import annotations import logging from dataclasses import asdict +from apps.core.tasks import PeriodicTask as CorePeriodicTask from celery import shared_task +from django.conf import settings +from django.core.cache import cache from organizations.cache import invalidate_organization_api_cache from organizations.source_backfill import OrganizationSourceBackfillService +from organizations.source_record_export import build_source_record_export_artifacts logger = logging.getLogger(__name__) +@shared_task(bind=True, base=CorePeriodicTask) +def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 + """Build and atomically publish the nightly source-record export matrix.""" + lock_key = getattr( + settings, + "SOURCE_RECORD_EXPORT_LOCK_KEY", + "organizations:source-record-exports:lock", + ) + lock_ttl = int( + getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60) + ) + if not cache.add(lock_key, "1", timeout=lock_ttl): + logger.info("Source-record export generation skipped: lock is already held") + return {"status": "skipped", "reason": "locked"} + + try: + generation = build_source_record_export_artifacts() + result = { + "status": "success", + "generation_id": generation.generation_id, + "generated_at": generation.generated_at, + "export_year": generation.export_year, + "artifacts_count": generation.artifacts_count, + "files_count": generation.files_count, + "records_count": generation.records_count, + "total_size": generation.total_size, + } + logger.info("Source-record export generation published: %s", result) + return result + finally: + cache.delete(lock_key) + + @shared_task def backfill_all_organization_sources(batch_size: int = 100) -> dict: """Backfill all organization source extensions from legacy parser tables.""" diff --git a/src/organizations/test_companies.py b/src/organizations/test_companies.py index 8a48023..b6bfdc2 100644 --- a/src/organizations/test_companies.py +++ b/src/organizations/test_companies.py @@ -10,6 +10,7 @@ from uuid import UUID, uuid5 from apps.parsers.models import ( FinancialReport, + FinancialReportLine, GenericParserRecord, IndustrialCertificateRecord, IndustrialProductRecord, @@ -32,9 +33,80 @@ from organizations.source_cache import invalidate_source_data_cache from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS TEST_COMPANY_COUNT = 20 +TEST_FINANCIAL_HISTORY_YEARS = 4 TEST_COMPANY_NAMESPACE = UUID("59b36ae9-bcf8-4b7c-b77a-77578f485a01") TEST_RECORD_PREFIX = "mostovik-test-company" +TEST_BALANCE_LINE_NAMES = { + "1110": "Нематериальные активы", + "1120": "Результаты исследований и разработок", + "1130": "Нематериальные поисковые активы", + "1140": "Материальные поисковые активы", + "1150": "Основные средства", + "1160": "Доходные вложения в материальные ценности", + "1170": "Финансовые вложения", + "1180": "Отложенные налоговые активы", + "1190": "Прочие внеоборотные активы", + "1100": "Итого по разделу I", + "1210": "Запасы", + "1220": "НДС по приобретенным ценностям", + "1230": "Дебиторская задолженность", + "1240": "Финансовые вложения (за исключением денежных эквивалентов)", + "1250": "Денежные средства и денежные эквиваленты", + "1260": "Прочие оборотные активы", + "1200": "Итого по разделу II", + "1600": "БАЛАНС (актив)", + "1310": "Уставный капитал", + "1320": "Собственные акции, выкупленные у акционеров", + "1340": "Переоценка внеоборотных активов", + "1350": "Добавочный капитал (без переоценки)", + "1360": "Резервный капитал", + "1370": "Нераспределенная прибыль (непокрытый убыток)", + "1300": "Итого по разделу III", + "1410": "Заемные средства", + "1420": "Отложенные налоговые обязательства", + "1430": "Оценочные обязательства", + "1450": "Прочие обязательства", + "1400": "Итого по разделу IV", + "1510": "Заемные средства", + "1520": "Кредиторская задолженность", + "1530": "Доходы будущих периодов", + "1540": "Оценочные обязательства", + "1550": "Прочие обязательства", + "1500": "Итого по разделу V", + "1700": "БАЛАНС (пассив)", +} + +TEST_PROFIT_LOSS_LINE_NAMES = { + "2110": "Выручка", + "2120": "Себестоимость продаж", + "2100": "Валовая прибыль (убыток)", + "2210": "Коммерческие расходы", + "2220": "Управленческие расходы", + "2200": "Прибыль (убыток) от продаж", + "2310": "Доходы от участия в других организациях", + "2320": "Проценты к получению", + "2330": "Проценты к уплате", + "2340": "Прочие доходы", + "2350": "Прочие расходы", + "2300": "Прибыль (убыток) до налогообложения", + "2410": "Налог на прибыль", + "2411": "Текущий налог на прибыль", + "2412": "Отложенный налог на прибыль", + "2460": "Прочее", + "2400": "Чистая прибыль (убыток)", + "2510": "Результат от переоценки внеоборотных активов", + "2520": "Результат от прочих операций", + "2530": "Налог на прибыль от операций вне чистой прибыли", + "2500": "Совокупный финансовый результат периода", + "2900": "Базовая прибыль (убыток) на акцию", + "2910": "Разводненная прибыль (убыток) на акцию", +} + +TEST_FINANCIAL_LINES_PER_YEAR = len(TEST_BALANCE_LINE_NAMES) + len( + TEST_PROFIT_LOSS_LINE_NAMES +) + @dataclass(frozen=True) class TestCompanyDatasetResult: @@ -128,9 +200,32 @@ class TestCompanyDatasetService: f"{legacy_module}.{legacy_record.__class__.__name__}" ) record.legacy_pk = str(legacy_record.pk) + legacy_owner = ( + OrganizationSourceRecord.objects.filter( + legacy_model=record.legacy_model, + legacy_pk=record.legacy_pk, + ) + .exclude(pk=record.pk) + .select_related("extension") + .first() + ) + if legacy_owner is not None: + if ( + legacy_owner.source != source + or legacy_owner.extension.organization_id != organization.pk + ): + raise RuntimeError( + "Legacy parser record is already linked to another " + "organization source record" + ) + legacy_owner.delete() record.save(update_fields=["legacy_model", "legacy_pk", "updated_at"]) if source == ParserLoadLog.Source.FNS_REPORTS: - cls._refresh_financial_lines(record=record, index=index) + cls._refresh_financial_lines( + record=record, + legacy_report=legacy_record, + index=index, + ) OrganizationSourceRecord.objects.filter( extension__organization=organization, @@ -288,12 +383,12 @@ class TestCompanyDatasetService: if source == ParserLoadLog.Source.FNS_REPORTS: legacy_record, _ = FinancialReport.objects.update_or_create( - external_id=f"test-company-{index:02d}", + file_hash=payload["file_hash"], defaults={ **common, + "external_id": external_id, "ogrn": organization.ogrn, "file_name": payload["file_name"], - "file_hash": payload["file_hash"], "status": FinancialReport.Status.SUCCESS, "source": FinancialReport.SourceType.API, }, @@ -423,6 +518,7 @@ class TestCompanyDatasetService: @classmethod def _record_defaults(cls, *, source: str, index: int, organization): + report_year = timezone.localdate().year common = { "inn": organization.inn, "ogrn": organization.ogrn, @@ -433,8 +529,11 @@ class TestCompanyDatasetService: url = f"https://example.test/{source}/{index}" values = { ParserLoadLog.Source.FNS_REPORTS: { - "title": f"Бухгалтерская отчетность за 2025 год — {organization.name}", - "record_date": "2025", + "title": ( + f"Бухгалтерская отчетность за {report_year} год — " + f"{organization.name}" + ), + "record_date": str(report_year), "status": "processed", "payload": { **common, @@ -442,7 +541,9 @@ class TestCompanyDatasetService: "file_hash": sha256( f"financial:{organization.ogrn}".encode() ).hexdigest(), - "lines_count": 4, + "lines_count": ( + TEST_FINANCIAL_LINES_PER_YEAR * TEST_FINANCIAL_HISTORY_YEARS + ), }, }, ParserLoadLog.Source.PROCUREMENTS: cls._procurement_values( @@ -660,32 +761,240 @@ class TestCompanyDatasetService: } @staticmethod - def _refresh_financial_lines(*, record, index: int) -> None: - expected = { - ("1", "1600", "Баланс (актив)", 10_000_000 + index * 100_000), - ("1", "1300", "Капитал и резервы", 4_000_000 + index * 50_000), - ("2", "2110", "Выручка", 20_000_000 + index * 200_000), - ("2", "2400", "Чистая прибыль", 2_000_000 + index * 25_000), - } - expected_keys = [] - for form_code, line_code, line_name, period_end in expected: - expected_keys.append((form_code, line_code, 2025)) - OrganizationSourceFinancialLine.objects.update_or_create( - source_record=record, - form_code=form_code, - line_code=line_code, - year=2025, - defaults={ - "line_name": line_name, - "period_start": period_end - 100_000, - "period_end": period_end, - }, + def _test_amount( + *, + base: int, + index: int, + year: int, + per_company: int, + annual_growth: int, + ) -> int: + """Return a stable amount for the same company and calendar year.""" + return base + index * per_company + (year - 2020) * annual_growth + + @classmethod + def _balance_values(cls, *, index: int, year: int) -> dict[str, int]: + """Build a balanced form 0710001 in thousands of rubles.""" + + def amount(base: int, per_company: int, annual_growth: int) -> int: + return cls._test_amount( + base=base, + index=index, + year=year, + per_company=per_company, + annual_growth=annual_growth, ) + + values = { + "1110": amount(350, 7, 25), + "1120": amount(220, 5, 18), + "1130": amount(80, 2, 6), + "1140": amount(70, 2, 5), + "1150": amount(4_800, 60, 240), + "1160": amount(300, 8, 22), + "1170": amount(900, 18, 70), + "1180": amount(180, 4, 12), + "1190": amount(140, 3, 10), + "1210": amount(3_400, 45, 190), + "1220": amount(240, 5, 12), + "1230": amount(3_800, 55, 220), + "1240": amount(1_100, 20, 90), + "1250": amount(1_500, 30, 120), + "1260": amount(260, 6, 16), + "1310": amount(3_500, 30, 100), + "1320": -amount(120, 2, 5), + "1340": amount(320, 4, 15), + "1350": amount(280, 3, 12), + "1360": amount(240, 3, 10), + "1410": amount(2_300, 32, 110), + "1420": amount(220, 4, 12), + "1430": amount(180, 3, 9), + "1450": amount(240, 4, 11), + "1510": amount(1_600, 25, 80), + "1520": amount(3_200, 40, 150), + "1530": amount(120, 2, 6), + "1540": amount(250, 4, 12), + "1550": amount(320, 5, 15), + } + + values["1100"] = sum( + values[code] + for code in ( + "1110", + "1120", + "1130", + "1140", + "1150", + "1160", + "1170", + "1180", + "1190", + ) + ) + values["1200"] = sum( + values[code] for code in ("1210", "1220", "1230", "1240", "1250", "1260") + ) + values["1600"] = values["1100"] + values["1200"] + values["1400"] = sum(values[code] for code in ("1410", "1420", "1430", "1450")) + values["1500"] = sum( + values[code] for code in ("1510", "1520", "1530", "1540", "1550") + ) + equity_without_retained = sum( + values[code] for code in ("1310", "1320", "1340", "1350", "1360") + ) + values["1370"] = ( + values["1600"] - equity_without_retained - values["1400"] - values["1500"] + ) + values["1300"] = equity_without_retained + values["1370"] + values["1700"] = values["1300"] + values["1400"] + values["1500"] + return values + + @classmethod + def _profit_loss_values(cls, *, index: int, year: int) -> dict[str, int]: + """Build a form 0710002 whose calculated rows reconcile.""" + + def amount(base: int, per_company: int, annual_growth: int) -> int: + return cls._test_amount( + base=base, + index=index, + year=year, + per_company=per_company, + annual_growth=annual_growth, + ) + + revenue = amount(32_000, 800, 2_400) + cost_of_sales = revenue * 61 // 100 + gross_profit = revenue - cost_of_sales + commercial_expenses = revenue * 5 // 100 + management_expenses = revenue * 8 // 100 + sales_profit = gross_profit - commercial_expenses - management_expenses + participation_income = amount(180, 6, 14) + interest_income = amount(220, 5, 16) + interest_expense = amount(310, 7, 20) + other_income = amount(760, 15, 45) + other_expense = amount(620, 12, 38) + profit_before_tax = ( + sales_profit + + participation_income + + interest_income + - interest_expense + + other_income + - other_expense + ) + current_tax = profit_before_tax * 19 // 100 + deferred_tax = profit_before_tax // 100 + income_tax = -current_tax + deferred_tax + other_tax_effects = -amount(25, 1, 2) + net_profit = profit_before_tax + income_tax + other_tax_effects + revaluation_result = amount(90, 3, 7) + other_operations_result = amount(55, 2, 5) + other_operations_tax = -amount(18, 1, 1) + comprehensive_result = ( + net_profit + + revaluation_result + + other_operations_result + + other_operations_tax + ) + basic_eps = max(1, net_profit // 1_000) + diluted_eps = max(1, basic_eps - 1) + + return { + "2110": revenue, + "2120": cost_of_sales, + "2100": gross_profit, + "2210": commercial_expenses, + "2220": management_expenses, + "2200": sales_profit, + "2310": participation_income, + "2320": interest_income, + "2330": interest_expense, + "2340": other_income, + "2350": other_expense, + "2300": profit_before_tax, + "2410": income_tax, + "2411": current_tax, + "2412": deferred_tax, + "2460": other_tax_effects, + "2400": net_profit, + "2510": revaluation_result, + "2520": other_operations_result, + "2530": other_operations_tax, + "2500": comprehensive_result, + "2900": basic_eps, + "2910": diluted_eps, + } + + @classmethod + def _financial_line_definitions(cls, *, index: int, year: int): + forms = ( + ( + "1", + TEST_BALANCE_LINE_NAMES, + cls._balance_values, + ), + ( + "2", + TEST_PROFIT_LOSS_LINE_NAMES, + cls._profit_loss_values, + ), + ) + for form_code, names, value_factory in forms: + period_start_values = value_factory(index=index, year=year - 1) + period_end_values = value_factory(index=index, year=year) + for line_code, line_name in names.items(): + yield ( + form_code, + line_code, + line_name, + period_start_values[line_code], + period_end_values[line_code], + ) + + @classmethod + def _refresh_financial_lines(cls, *, record, legacy_report, index: int) -> None: + current_year = timezone.localdate().year + report_years = range( + current_year - TEST_FINANCIAL_HISTORY_YEARS + 1, + current_year + 1, + ) + expected_keys = set() + for report_year in report_years: + for ( + form_code, + line_code, + line_name, + period_start, + period_end, + ) in cls._financial_line_definitions(index=index, year=report_year): + expected_keys.add((form_code, line_code, report_year)) + defaults = { + "line_name": line_name, + "period_start": period_start, + "period_end": period_end, + } + OrganizationSourceFinancialLine.objects.update_or_create( + source_record=record, + form_code=form_code, + line_code=line_code, + year=report_year, + defaults=defaults, + ) + FinancialReportLine.objects.update_or_create( + report=legacy_report, + form_code=form_code, + line_code=line_code, + year=report_year, + defaults=defaults, + ) lines = OrganizationSourceFinancialLine.objects.filter(source_record=record) for line in lines: key = (line.form_code, line.line_code, line.year) if key not in expected_keys: line.delete() + for line in legacy_report.lines.all(): + key = (line.form_code, line.line_code, line.year) + if key not in expected_keys: + line.delete() @staticmethod def _refresh_extension_counters(*, organization) -> None: diff --git a/src/organizations/views.py b/src/organizations/views.py index acb1583..0cce862 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -16,7 +16,7 @@ from django.core.cache import cache from django.db.models import Case, CharField, F, Q, Value, When from django.db.models.fields.json import KeyTextTransform from django.db.models.functions import Cast, Coalesce, NullIf -from django.http import HttpResponse +from django.http import StreamingHttpResponse from django_filters import rest_framework as filters from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -51,11 +51,19 @@ from organizations.serializers import ( OrganizationDirectoryImportUploadSerializer, OrganizationSerializer, OrganizationSourceExtensionSerializer, + OrganizationSourceRecordExportDownloadSerializer, OrganizationSourceRecordExportRequestSerializer, OrganizationSourceRecordListResponseSerializer, OrganizationSourceRecordSerializer, ) -from organizations.source_record_export import build_source_records_export_archive +from organizations.source_record_export import ( + SourceRecordExportArchive, + SourceRecordExportArtifactsUnavailable, + SourceRecordExportTicketInvalid, + build_source_records_export_archive, + consume_source_record_export_download_ticket, + create_source_record_export_download_ticket, +) ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations") @@ -536,12 +544,29 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): ordering = ["-created_at", "-uid"] def get_permissions(self): - if self.action == "export": + if self.action in {"export", "export_ticket"}: return [IsAdminUser()] + if self.action == "export_download": + return [AllowAny()] if getattr(settings, "ORGANIZATIONS_V2_ALLOW_ANONYMOUS", False): return [AllowAny()] return super().get_permissions() + @staticmethod + def _source_record_export_response( + package: SourceRecordExportArchive, + ) -> StreamingHttpResponse: + response = StreamingHttpResponse( + package.archive_chunks, + content_type="application/zip", + ) + response[ + "Content-Disposition" + ] = f'attachment; filename="{package.archive_name}"' + response["X-Source-Export-Files"] = str(package.files_count) + response["X-Source-Export-Generated-At"] = package.generated_at + return response + def get_queryset(self): raw_record_date = NullIf(F("record_date"), Value("")) inspection_record_date = Coalesce( @@ -732,7 +757,8 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): operation_id="v2_organization_source_records_export", operation_summary="Выгрузить записи источников", operation_description=( - "Формирует ZIP-архив с одним файлом на выбранную группу источников. " + "Упаковывает в ZIP ночные готовые файлы выбранных групп источников. " + "При скачивании данные из БД повторно не формируются. " "Финансово-экономические показатели всегда выгружаются в JSON." ), request_body=OrganizationSourceRecordExportRequestSerializer, @@ -743,22 +769,136 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): ), 400: "Некорректные параметры выгрузки.", 403: "Доступ разрешён только администраторам.", + 503: "Ночная выгрузка ещё не сформирована.", }, ) @action(detail=False, methods=["post"], url_path="export") - def export(self, request, *args: Any, **kwargs: Any) -> HttpResponse: + def export( + self, + request, + *args: Any, + **kwargs: Any, + ) -> StreamingHttpResponse | Response: serializer = OrganizationSourceRecordExportRequestSerializer(data=request.data) serializer.is_valid(raise_exception=True) - package = build_source_records_export_archive( - source_groups=serializer.validated_data["sources"], - export_format=serializer.validated_data["format"], + try: + package = build_source_records_export_archive( + source_groups=serializer.validated_data["sources"], + export_format=serializer.validated_data["format"], + ) + except SourceRecordExportArtifactsUnavailable: + return Response( + { + "detail": "Готовая ночная выгрузка ещё не сформирована.", + "code": "source_export_not_ready", + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "3600"}, + ) + + return self._source_record_export_response(package) + + @swagger_auto_schema( + tags=[ORGANIZATIONS_TAG], + operation_id="v2_organization_source_records_export_ticket", + operation_summary="Подготовить нативное скачивание записей источников", + operation_description=( + "Возвращает короткоживущий одноразовый ticket. Frontend отправляет " + "его обычной HTML-формой в export-download, чтобы браузер сохранял " + "потоковый ZIP напрямую на диск без Blob в JavaScript." + ), + request_body=OrganizationSourceRecordExportRequestSerializer, + responses={ + 201: "Одноразовый ticket и имя ZIP-файла.", + 400: "Некорректные параметры выгрузки.", + 403: "Доступ разрешён только администраторам.", + 503: "Ночная выгрузка ещё не сформирована.", + }, + ) + @action(detail=False, methods=["post"], url_path="export-ticket") + def export_ticket(self, request, *args: Any, **kwargs: Any) -> Response: + serializer = OrganizationSourceRecordExportRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + download_ticket = create_source_record_export_download_ticket( + source_groups=serializer.validated_data["sources"], + export_format=serializer.validated_data["format"], + ) + except SourceRecordExportArtifactsUnavailable: + return Response( + { + "detail": "Готовая ночная выгрузка ещё не сформирована.", + "code": "source_export_not_ready", + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "3600"}, + ) + except RuntimeError: + return Response( + { + "detail": "Не удалось подготовить скачивание. Повторите запрос.", + "code": "source_export_ticket_unavailable", + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "5"}, + ) + + return Response( + { + "ticket": download_ticket.ticket, + "file_name": download_ticket.archive_name, + "expires_in": download_ticket.expires_in, + }, + status=status.HTTP_201_CREATED, ) - response = HttpResponse(package.archive_bytes, content_type="application/zip") - response.status_code = status.HTTP_200_OK - response[ - "Content-Disposition" - ] = f'attachment; filename="{package.archive_name}"' - response["Content-Length"] = str(len(package.archive_bytes)) - response["X-Source-Export-Files"] = str(package.files_count) - return response + + @swagger_auto_schema( + tags=[ORGANIZATIONS_TAG], + operation_id="v2_organization_source_records_export_download", + operation_summary="Скачать готовые записи источников по ticket", + request_body=OrganizationSourceRecordExportDownloadSerializer, + responses={ + 200: openapi.Response( + description="Потоковый ZIP-архив записей источников.", + schema=openapi.Schema(type=openapi.TYPE_FILE), + ), + 400: "Ticket отсутствует или имеет неверный формат.", + 410: "Ticket истёк или уже использован.", + 503: "Опубликованная выгрузка больше недоступна.", + }, + ) + @action(detail=False, methods=["post"], url_path="export-download") + def export_download( + self, + request, + *args: Any, + **kwargs: Any, + ) -> StreamingHttpResponse | Response: + serializer = OrganizationSourceRecordExportDownloadSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + try: + package = consume_source_record_export_download_ticket( + serializer.validated_data["ticket"] + ) + except SourceRecordExportTicketInvalid: + return Response( + { + "detail": "Ticket скачивания истёк или уже использован.", + "code": "source_export_ticket_invalid", + }, + status=status.HTTP_410_GONE, + ) + except SourceRecordExportArtifactsUnavailable: + return Response( + { + "detail": "Опубликованная выгрузка больше недоступна.", + "code": "source_export_not_ready", + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "3600"}, + ) + + return self._source_record_export_response(package) diff --git a/src/settings/base.py b/src/settings/base.py index 9322919..a59e6d0 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -239,12 +239,32 @@ STATE_CORP_EXCHANGE_KEY_ID = os.getenv( "STATE_CORP_EXCHANGE_KEY_ID", "state-corp-shared-token" ).strip() STATE_CORP_EXCHANGE_TIMEOUT_SECONDS = int( - os.getenv("STATE_CORP_EXCHANGE_TIMEOUT_SECONDS", "60") + os.getenv("STATE_CORP_EXCHANGE_TIMEOUT_SECONDS", "300") ) BACKUP_EXPORT_DIRECTORY = os.getenv( "BACKUP_EXPORT_DIRECTORY", str(PROJECT_ROOT / "media" / "backups"), ) +SOURCE_RECORD_EXPORT_DIRECTORY = os.getenv( + "SOURCE_RECORD_EXPORT_DIRECTORY", + str(PROJECT_ROOT / "media" / "source-record-exports"), +) +SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP = int( + os.getenv("SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP", "2") +) +SOURCE_RECORD_EXPORT_LOCK_KEY = os.getenv( + "SOURCE_RECORD_EXPORT_LOCK_KEY", + "organizations:source-record-exports:lock", +) +SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS = int( + os.getenv("SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", str(6 * 60 * 60)) +) +SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE = int( + os.getenv("SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE", "100000") +) +SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS = int( + os.getenv("SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS", "300") +) # Celery: сохраняем ретраи подключения на старте и для 6.x совместимости. CELERY_BROKER_CONNECTION_RETRY = True @@ -453,7 +473,7 @@ FNS_PROCESSED_DIRECTORY = PROJECT_ROOT / "input" / "fns" / "processed" FNS_FAILED_DIRECTORY = PROJECT_ROOT / "input" / "fns" / "failed" # ============================================================================= -# Checko API Settings (checko.ru) +# External organization data API settings # ============================================================================= CHECKO_API_KEY = os.getenv("CHECKO_API_KEY", "") diff --git a/tests/apps/core/test_background_jobs.py b/tests/apps/core/test_background_jobs.py index a56d022..9ee83b8 100644 --- a/tests/apps/core/test_background_jobs.py +++ b/tests/apps/core/test_background_jobs.py @@ -61,6 +61,20 @@ class BackgroundJobModelTest(TestCase): self.assertEqual(job.result, result) self.assertIsNotNone(job.completed_at) + def test_complete_clears_previous_failure_details(self): + job = BackgroundJob.objects.create( + task_id=fake.uuid4(), + task_name="test.task", + error="stale error", + traceback="stale traceback", + ) + + job.complete(result={"processed": 1}) + + self.assertEqual(job.status, JobStatus.SUCCESS) + self.assertEqual(job.error, "") + self.assertEqual(job.traceback, "") + def test_fail(self): """Тест завершения с ошибкой.""" job = BackgroundJob.objects.create( @@ -288,7 +302,7 @@ class BackgroundJobServiceTest(TestCase): task_name="apps.other.tasks.task", meta={"source": "industrial_products"}, ) - old_timestamp = timezone.now() - timedelta(hours=3) + old_timestamp = timezone.now() - timedelta(hours=25) BackgroundJob.objects.filter( task_id__in=[stale_job.task_id, unrelated_job.task_id] ).update(created_at=old_timestamp, updated_at=timezone.now()) @@ -307,3 +321,37 @@ class BackgroundJobServiceTest(TestCase): self.assertIn("Stale background job", stale_job.error) self.assertEqual(fresh_job.status, JobStatus.PENDING) self.assertEqual(unrelated_job.status, JobStatus.PENDING) + + def test_mark_stale_active_jobs_uses_updated_at_for_started_job(self): + job = BackgroundJobService.create_job( + task_id="job-heartbeat", + task_name="apps.parsers.tasks.parse_industrial_products", + meta={"source": "industrial_products"}, + ) + job.mark_started() + old_timestamp = timezone.now() - timedelta(hours=5) + BackgroundJob.objects.filter(pk=job.pk).update( + started_at=old_timestamp, + updated_at=timezone.now(), + ) + + updated = BackgroundJobService.mark_stale_active_jobs_failed( + max_age_minutes=240, + task_names={"apps.parsers.tasks.parse_industrial_products"}, + meta_sources={"industrial_products"}, + ) + + self.assertEqual(updated, 0) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.STARTED) + + BackgroundJob.objects.filter(pk=job.pk).update(updated_at=old_timestamp) + updated = BackgroundJobService.mark_stale_active_jobs_failed( + max_age_minutes=240, + task_names={"apps.parsers.tasks.parse_industrial_products"}, + meta_sources={"industrial_products"}, + ) + + self.assertEqual(updated, 1) + job.refresh_from_db() + self.assertEqual(job.status, JobStatus.FAILURE) diff --git a/tests/apps/exchange/test_service_units.py b/tests/apps/exchange/test_service_units.py index fc73e14..2e7d985 100644 --- a/tests/apps/exchange/test_service_units.py +++ b/tests/apps/exchange/test_service_units.py @@ -173,8 +173,16 @@ class ExchangeConnectionServiceUnitTest(TestCase): def test_test_connection_payload_does_not_persist_connection(self): with patch.object( ExchangeConnectionService, - "validate_saved_connection", - ) as validate_mock: + "test_connection", + return_value="target_alias", + ) as test_connection_mock, patch.object( + ExchangeConnectionService, + "_cleanup_alias", + ) as cleanup_mock, patch.object( + ExchangeConnectionService, + "_validate_schema_exists", + side_effect=AssertionError("test-only connection check must not validate schema"), + ): result = ExchangeConnectionService.test_connection_payload( server="127.0.0.1", port=5432, @@ -187,10 +195,11 @@ class ExchangeConnectionServiceUnitTest(TestCase): self.assertEqual(result["status"], "success") self.assertEqual( result["message"], - "Подключение проверено. Соединение и структура БД валидны.", + "Подключение проверено. PostgreSQL доступен, логин и пароль верны.", ) self.assertEqual(ExchangeConnection.objects.count(), 0) - validate_mock.assert_called_once() + test_connection_mock.assert_called_once() + cleanup_mock.assert_called_once_with("target_alias") def test_get_active_connection_raises_when_missing(self): with self.assertRaisesMessage( @@ -496,6 +505,27 @@ class ExchangeConnectionServiceUnitTest(TestCase): self.assertEqual(connections_mock.databases[alias]["PASSWORD"], "secret") self.assertNotIn(alias, storage.__dict__) + def test_configure_alias_uses_unique_alias_for_unsaved_connection(self): + connection = ExchangeConnection( + server="127.0.0.1", + port=5432, + username="postgres", + password="secret", # noqa: S106 + database_name="target_db", + schema_name="public", + ) + + first_alias = ExchangeConnectionService._configure_alias(connection) + second_alias = ExchangeConnectionService._configure_alias(connection) + + try: + self.assertNotEqual(first_alias, second_alias) + self.assertTrue(first_alias.startswith("exchange_target_")) + self.assertTrue(second_alias.startswith("exchange_target_")) + finally: + ExchangeConnectionService._cleanup_alias(first_alias) + ExchangeConnectionService._cleanup_alias(second_alias) + def test_validate_schema_exists_raises_when_schema_missing(self): cursor = MagicMock() cursor.fetchone.return_value = None diff --git a/tests/apps/exchange/test_services.py b/tests/apps/exchange/test_services.py index a50bc1a..18b8149 100644 --- a/tests/apps/exchange/test_services.py +++ b/tests/apps/exchange/test_services.py @@ -5,9 +5,9 @@ from contextlib import suppress from apps.exchange.models import ExchangeConnection from apps.exchange.services import ExchangeConnectionService from apps.parsers.models import IndustrialCertificateRecord, ParserLoadLog -from apps.registers.models import Organization from django.db import connections from django.test import TestCase +from organizations.models import Organization class ExchangeConnectionServiceDependenciesTest(TestCase): @@ -20,7 +20,7 @@ class ExchangeConnectionServiceDependenciesTest(TestCase): self.assertEqual(models_to_copy, [ParserLoadLog]) - def test_extend_models_adds_registers_organization_first(self): + def test_extend_models_adds_fk_target_organization_first(self): models_to_copy = ExchangeConnectionService._extend_models_with_dependencies( [IndustrialCertificateRecord] ) diff --git a/tests/apps/exchange/test_state_corp_services.py b/tests/apps/exchange/test_state_corp_services.py index 7b60287..30483a7 100644 --- a/tests/apps/exchange/test_state_corp_services.py +++ b/tests/apps/exchange/test_state_corp_services.py @@ -249,10 +249,12 @@ class StateCorpExchangeServiceTest(TestCase): ogrn=organization.ogrn, title="А40-1/2026", record_date="2026-03-25", + amount="1250000.50", status="in_progress", payload={ "case_number": "А40-1/2026", "court_name": "АС города Москвы", + "claim_amount": "1.00", "target": {"role": "ответчик"}, }, ) @@ -429,6 +431,10 @@ class StateCorpExchangeServiceTest(TestCase): payload["data"]["arbitration_cases"][0]["case_number"], "А40-1/2026", ) + self.assertEqual( + payload["data"]["arbitration_cases"][0]["claim_amount"], + "1250000.50", + ) self.assertEqual( payload["data"]["bankruptcy_procedures"][0]["case_number"], "А40-555/2026", @@ -541,3 +547,4 @@ class StateCorpExchangeServiceTest(TestCase): TEST_STATE_CORP_TOKEN, ) self.assertEqual(kwargs["files"]["file"][0], package.archive_name) + self.assertEqual(kwargs["timeout"], 300) diff --git a/tests/apps/organizations/test_api_v2_source_extensions.py b/tests/apps/organizations/test_api_v2_source_extensions.py index 3a6740b..5e5bf86 100644 --- a/tests/apps/organizations/test_api_v2_source_extensions.py +++ b/tests/apps/organizations/test_api_v2_source_extensions.py @@ -170,6 +170,59 @@ class OrganizationSourceExtensionsApiV2Test(APITestCase): self.assertEqual(record["source_group"], "planned_inspections") self.assertEqual(record["organization"]["uid"], str(target.uid)) + def test_flat_arbitration_records_expose_frontend_role_labels(self): + organization = create_frontend_organization( + name='ООО "Arbitration Roles"', + inn="7707083899", + ogrn="1027700132099", + ) + extension = ArbitrationExtension.objects.create( + organization=organization, + title="Арбитраж", + ) + expected_roles = { + "ROLE-PLAINTIFF": "Истец", + "ROLE-DEFENDANT": "Ответчик", + "ROLE-THIRD-PARTY": "Третье лицо", + } + for external_id, provider_role in ( + ("ROLE-PLAINTIFF", "plaintiff"), + ("ROLE-DEFENDANT", "defendant"), + ("ROLE-THIRD-PARTY", "third_party"), + ): + OrganizationSourceRecord.objects.create( + extension=extension, + record_type="arbitration_case", + source="arbitration", + external_id=external_id, + payload={"target": {"role": provider_role}}, + ) + + response = self.client.get( + reverse("api_v2:organizations:organization-source-records-list"), + { + "source_group": "arbitration", + "source": "arbitration", + "organization": str(organization.uid), + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + records_by_external_id = { + item["external_id"]: item for item in response.data["data"] + } + self.assertEqual( + { + external_id: records_by_external_id[external_id]["payload"]["role"] + for external_id in expected_roles + }, + expected_roles, + ) + self.assertNotIn( + "role", + OrganizationSourceRecord.objects.get(external_id="ROLE-PLAINTIFF").payload, + ) + def test_flat_source_records_filters_supported_groups_by_canonical_date(self): organization = create_frontend_organization( name='ООО "Canonical dates"', diff --git a/tests/apps/organizations/test_restore_fns_financial_lines.py b/tests/apps/organizations/test_restore_fns_financial_lines.py new file mode 100644 index 0000000..f680b02 --- /dev/null +++ b/tests/apps/organizations/test_restore_fns_financial_lines.py @@ -0,0 +1,95 @@ +"""Tests for restoring normalized FNS financial lines.""" + +from __future__ import annotations + +from pathlib import Path +from tempfile import TemporaryDirectory + +from apps.parsers.models import ParserLoadLog +from django.core.management import call_command +from django.test import TestCase, override_settings +from openpyxl import Workbook +from organizations.models import ( + FinancialIndicatorsExtension, + OrganizationSourceFinancialLine, + OrganizationSourceRecord, +) + +from tests.apps.parsers.organization_helpers import create_directory_organization + + +class RestoreFNSFinancialLinesCommandTests(TestCase): + """Restore only empty reports and keep reruns idempotent.""" + + def test_dry_run_then_restores_codeless_processed_workbook(self): + ogrn = "1187700006273" + external_id = "0130160" + organization = create_directory_organization( + name="Тестовая организация", + inn="7700000001", + kpp="770001001", + okpo="12345678", + ogrn=ogrn, + ) + extension = FinancialIndicatorsExtension.objects.create( + organization=organization, + title="Финансово-экономические показатели", + ) + file_name = f"fin_{external_id}_{ogrn}.xlsx" + record = OrganizationSourceRecord.objects.create( + extension=extension, + source=ParserLoadLog.Source.FNS_REPORTS, + record_type="financial_report", + external_id=external_id, + title=file_name, + status="success", + payload={ + "external_id": external_id, + "file_name": file_name, + "lines_count": 0, + "ogrn": ogrn, + }, + ) + + with TemporaryDirectory() as temp_directory: + workbook = Workbook() + worksheet = workbook.active + worksheet.append(["Форма №1", 2020, None, 2021, None]) + worksheet.append( + ["Бухгалтерский баланс", "Начало", "Конец", "Начало", "Конец"] + ) + worksheet.append(["Баланс", 60_232, 234_841, 234_841, 244_479]) + workbook.save(Path(temp_directory) / file_name) + + with override_settings(FNS_PROCESSED_DIRECTORY=temp_directory): + call_command( + "restore_fns_financial_lines", + "--ogrn", + ogrn, + "--dry-run", + "--silent", + ) + self.assertFalse( + OrganizationSourceFinancialLine.objects.filter( + source_record=record + ).exists() + ) + + call_command( + "restore_fns_financial_lines", + "--ogrn", + ogrn, + "--silent", + ) + call_command( + "restore_fns_financial_lines", + "--ogrn", + ogrn, + "--silent", + ) + + lines = OrganizationSourceFinancialLine.objects.filter(source_record=record) + self.assertEqual(lines.count(), 2) + self.assertEqual(set(lines.values_list("year", flat=True)), {2020, 2021}) + record.refresh_from_db() + self.assertEqual(record.payload["lines_count"], 2) diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index 2217489..71b8c54 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -2,10 +2,18 @@ import csv import json +import os import zipfile +from datetime import UTC, datetime from io import BytesIO, StringIO +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch +from django.core.management import call_command +from django.test import override_settings from django.urls import reverse +from django.utils import timezone from openpyxl import load_workbook from organizations.models import ( FinancialIndicatorsExtension, @@ -15,6 +23,15 @@ from organizations.models import ( PlannedInspectionExtension, SourceGroup, ) +from organizations.source_record_export import ( + SourceRecordExportArtifactsUnavailable, + _render_source_group_artifact, + _source_group_queryset, + _spool_source_group_rows, + build_source_record_export_artifacts, + build_source_records_export_archive, + load_current_source_record_export_generation, +) from rest_framework import status from rest_framework.test import APITestCase @@ -25,9 +42,169 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): """Checks admin-only source-record export contract.""" def setUp(self): + self.export_directory = TemporaryDirectory() + self.settings_override = override_settings( + SOURCE_RECORD_EXPORT_DIRECTORY=self.export_directory.name, + SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP=2, + ) + self.settings_override.enable() self.url = reverse( "api_v2:organizations:organization-source-records-export", ) + self.ticket_url = reverse( + "api_v2:organizations:organization-source-records-export-ticket", + ) + self.download_url = reverse( + "api_v2:organizations:organization-source-records-export-download", + ) + + def tearDown(self): + self.settings_override.disable() + self.export_directory.cleanup() + super().tearDown() + + @staticmethod + def _response_body(response) -> bytes: + if response.streaming: + return b"".join(response.streaming_content) + return response.content + + def test_export_returns_service_unavailable_before_first_nightly_generation(self): + self.client.force_authenticate(UserFactory.create_superuser()) + + response = self.client.post( + self.url, + { + "sources": [SourceGroup.PLANNED_INSPECTIONS.value], + "format": "json", + }, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE) + self.assertEqual(response.data["code"], "source_export_not_ready") + self.assertEqual(response["Retry-After"], "3600") + + def test_management_command_bootstraps_first_generation(self): + command_output = StringIO() + + call_command("build_source_record_exports", stdout=command_output) + + generation = load_current_source_record_export_generation() + self.assertEqual(generation.artifacts_count, 25) + self.assertEqual(generation.export_year, timezone.localdate().year) + self.assertIn('"artifacts_count": 25', command_output.getvalue()) + + def test_generation_contains_only_records_from_its_calendar_year(self): + export_year = 2026 + generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC) + organization = Organization.objects.create( + name='ООО "Годовая выгрузка"', + inn="7707083899", + ) + inspection_extension = PlannedInspectionExtension.objects.create( + organization=organization, + title="Плановые проверки Генпрокуратуры России", + ) + current_record = OrganizationSourceRecord.objects.create( + extension=inspection_extension, + record_type="inspection", + source="inspections", + external_id="INSP-2026", + title="Текущая проверка", + record_date="15.02.2026", + ) + OrganizationSourceRecord.objects.create( + extension=inspection_extension, + record_type="inspection", + source="inspections", + external_id="INSP-2025", + title="Прошлогодняя проверка", + record_date="15.02.2025", + ) + dateless_current_record = OrganizationSourceRecord.objects.create( + extension=inspection_extension, + record_type="inspection", + source="inspections", + external_id="INSP-DATELESS-2026", + title="Запись без предметной даты", + ) + OrganizationSourceRecord.objects.filter(pk=dateless_current_record.pk).update( + created_at=generated_at + ) + + financial_extension = FinancialIndicatorsExtension.objects.create( + organization=organization, + title="Финансово-экономические показатели", + ) + current_financial_record = OrganizationSourceRecord.objects.create( + extension=financial_extension, + record_type="financial_report", + source="fns_reports", + external_id="FIN-CURRENT", + title="Отчёт с текущим годом", + ) + old_financial_record = OrganizationSourceRecord.objects.create( + extension=financial_extension, + record_type="financial_report", + source="fns_reports", + external_id="FIN-OLD", + title="Старый отчёт", + ) + for source_record, year in ( + (current_financial_record, 2025), + (current_financial_record, 2026), + (old_financial_record, 2025), + ): + OrganizationSourceFinancialLine.objects.create( + source_record=source_record, + form_code="1", + line_code=str(year), + line_name=f"Строка {year}", + year=year, + period_end=year, + ) + + generation = build_source_record_export_artifacts(now=generated_at) + + self.assertEqual(generation.export_year, export_year) + inspection_path = next( + artifact.path + for artifact in generation.artifacts + if artifact.source_group == SourceGroup.PLANNED_INSPECTIONS.value + and artifact.file_format == "json" + ) + inspection_rows = json.loads(inspection_path.read_text(encoding="utf-8")) + self.assertEqual( + {row["uid"] for row in inspection_rows}, + {str(current_record.pk), str(dateless_current_record.pk)}, + ) + + financial_path = next( + artifact.path + for artifact in generation.artifacts + if artifact.source_group == SourceGroup.FINANCIAL_INDICATORS.value + ) + financial_rows = json.loads(financial_path.read_text(encoding="utf-8")) + self.assertEqual( + [row["uid"] for row in financial_rows], + [str(current_financial_record.pk)], + ) + self.assertEqual( + {line["year"] for line in financial_rows[0]["financial_lines"]}, + {export_year}, + ) + + def test_new_calendar_year_requires_a_new_prepared_generation(self): + generated_at = datetime(2026, 12, 31, 23, 59, tzinfo=UTC) + build_source_record_export_artifacts(now=generated_at) + + with self.assertRaises(SourceRecordExportArtifactsUnavailable): + build_source_records_export_archive( + source_groups=[SourceGroup.PLANNED_INSPECTIONS.value], + export_format="json", + requested_at=datetime(2027, 1, 1, tzinfo=UTC), + ) def test_admin_exports_selected_sources_to_zip(self): self.client.force_authenticate(UserFactory.create_superuser()) @@ -37,6 +214,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): inn="7707083810", ogrn="1027700132010", kpp="770701001", + okpo="12345678", ) inspection_extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -70,31 +248,38 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): form_code="1", line_code="1600", line_name="Баланс", - year=2025, + year=timezone.localdate().year, period_start=100, period_end=200, ) + generation = build_source_record_export_artifacts() - response = self.client.post( - self.url, - { - "sources": [ - SourceGroup.PLANNED_INSPECTIONS.value, - SourceGroup.FINANCIAL_INDICATORS.value, - ], - "format": "xlsx", - }, - format="json", - ) + with self.assertNumQueries(0): + response = self.client.post( + self.url, + { + "sources": [ + SourceGroup.PLANNED_INSPECTIONS.value, + SourceGroup.FINANCIAL_INDICATORS.value, + ], + "format": "xlsx", + }, + format="json", + ) self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertTrue(response.streaming) self.assertEqual(response["Content-Type"], "application/zip") + self.assertEqual( + response["X-Source-Export-Generated-At"], generation.generated_at + ) + self.assertEqual(generation.artifacts_count, 25) self.assertIn( - 'filename="organization_source_records_export_', + 'filename="planned-inspections__financial-indicators_', response["Content-Disposition"], ) - with zipfile.ZipFile(BytesIO(response.content)) as archive: + with zipfile.ZipFile(BytesIO(self._response_body(response))) as archive: self.assertEqual( set(archive.namelist()), {"planned-inspections.xlsx", "financial-indicators.json"}, @@ -107,16 +292,17 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): worksheet = workbook["data"] rows = list(worksheet.iter_rows(values_only=True)) self.assertEqual( - rows[0][:4], - ("Наименование", "ИНН", "ОГРН", "КПП"), + rows[0][:5], + ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"), ) self.assertEqual( - rows[1][:4], + rows[1][:5], ( 'Общество с ограниченной ответственностью "Экспорт"', "7707083810", "1027700132010", "770701001", + "12345678", ), ) self.assertIn("payload.risk.score", rows[0]) @@ -131,6 +317,68 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): ) self.assertEqual(financial_rows[0]["financial_lines"][0]["period_end"], 200) + def test_admin_uses_one_time_ticket_for_native_zero_sql_download(self): + self.client.force_authenticate(UserFactory.create_superuser()) + requested_at = datetime(2026, 8, 4, 12, 34, 56, tzinfo=UTC) + build_source_record_export_artifacts(now=requested_at) + + with ( + patch( + "organizations.source_record_export.timezone.now", + return_value=requested_at, + ), + self.assertNumQueries(0), + ): + ticket_response = self.client.post( + self.ticket_url, + { + "sources": [SourceGroup.PLANNED_INSPECTIONS.value], + "format": "json", + }, + format="json", + ) + + self.assertEqual(ticket_response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ticket_response.data["expires_in"], 300) + self.assertRegex(ticket_response.data["ticket"], r"^[A-Za-z0-9_-]{43}$") + self.assertNotIn("download_url", ticket_response.data) + self.assertEqual( + ticket_response.data["file_name"], + "planned-inspections_20260804_123456.zip", + ) + + self.client.force_authenticate(user=None) + with self.assertNumQueries(0): + download_response = self.client.post( + self.download_url, + {"ticket": ticket_response.data["ticket"]}, + format="multipart", + ) + + self.assertEqual(download_response.status_code, status.HTTP_200_OK) + self.assertTrue(download_response.streaming) + self.assertEqual(download_response["Content-Type"], "application/zip") + self.assertNotIn("Content-Length", download_response) + self.assertEqual( + download_response["Content-Disposition"], + 'attachment; filename="planned-inspections_20260804_123456.zip"', + ) + with zipfile.ZipFile( + BytesIO(self._response_body(download_response)) + ) as archive: + self.assertEqual(archive.namelist(), ["planned-inspections.json"]) + + consumed_response = self.client.post( + self.download_url, + {"ticket": ticket_response.data["ticket"]}, + format="multipart", + ) + self.assertEqual(consumed_response.status_code, status.HTTP_410_GONE) + self.assertEqual( + consumed_response.data["code"], + "source_export_ticket_invalid", + ) + def test_csv_export_uses_bom_and_canonical_columns_before_payload(self): self.client.force_authenticate(UserFactory.create_superuser()) organization = Organization.objects.create( @@ -139,6 +387,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): inn="7707083811", ogrn="1027700132011", kpp="770701002", + okpo="87654321", ) extension = PlannedInspectionExtension.objects.create( organization=organization, @@ -152,6 +401,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): title="CSV проверка", payload={"nested": {"value": "данные"}}, ) + build_source_record_export_artifacts() response = self.client.post( self.url, @@ -164,18 +414,299 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) - with zipfile.ZipFile(BytesIO(response.content)) as archive: + with zipfile.ZipFile(BytesIO(self._response_body(response))) as archive: csv_bytes = archive.read("planned-inspections.csv") self.assertTrue(csv_bytes.startswith(b"\xef\xbb\xbf")) csv_text = csv_bytes.decode("utf-8-sig") csv_rows = list(csv.reader(StringIO(csv_text))) - self.assertEqual(csv_rows[0][:4], ["Наименование", "ИНН", "ОГРН", "КПП"]) self.assertEqual( - csv_rows[1][:4], ['ООО "CSV"', "7707083811", "1027700132011", "770701002"] + csv_rows[0][:5], + ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"], + ) + self.assertEqual( + csv_rows[1][:5], + [ + 'ООО "CSV"', + "7707083811", + "1027700132011", + "770701002", + "87654321", + ], ) self.assertIn("payload.nested.value", csv_rows[0]) + def test_xlsx_export_splits_rows_across_bounded_workbook_parts(self): + organization = Organization.objects.create( + name='ООО "Многолистовая выгрузка"', + inn="7707083812", + ) + extension = PlannedInspectionExtension.objects.create( + organization=organization, + title="Плановые проверки Генпрокуратуры России", + ) + for index in range(3): + OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="inspections", + external_id=f"INSP-SHEET-{index}", + title=f"Проверка {index}", + payload={}, + ) + + with override_settings(SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE=2): + generation = build_source_record_export_artifacts() + + artifacts = sorted( + ( + item + for item in generation.artifacts + if item.source_group == SourceGroup.PLANNED_INSPECTIONS.value + and item.file_format == "xlsx" + ), + key=lambda item: item.file_name, + ) + + self.assertEqual(generation.artifacts_count, 25) + self.assertEqual(generation.files_count, 26) + self.assertEqual( + [item.file_name for item in artifacts], + [ + "planned-inspections-part-001.xlsx", + "planned-inspections-part-002.xlsx", + ], + ) + first_workbook = load_workbook(artifacts[0].path, read_only=True) + second_workbook = load_workbook(artifacts[1].path, read_only=True) + + self.assertEqual(first_workbook.sheetnames, ["data"]) + self.assertEqual(second_workbook.sheetnames, ["data"]) + self.assertEqual( + len(list(first_workbook["data"].iter_rows(values_only=True))), + 3, + ) + self.assertEqual( + len(list(second_workbook["data"].iter_rows(values_only=True))), + 2, + ) + self.assertEqual( + next(second_workbook["data"].iter_rows(values_only=True))[:5], + ("Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"), + ) + + selected_artifacts = [ + item + for item in generation.artifacts + if item.source_group == SourceGroup.PLANNED_INSPECTIONS.value + ] + self.assertEqual(len(selected_artifacts), 4) + + package = build_source_records_export_archive( + source_groups=[SourceGroup.PLANNED_INSPECTIONS.value], + export_format="xlsx", + ) + archive_bytes = b"".join(package.archive_chunks) + + with zipfile.ZipFile(BytesIO(archive_bytes)) as archive: + self.assertEqual( + archive.namelist(), + [ + "planned-inspections-part-001.xlsx", + "planned-inspections-part-002.xlsx", + ], + ) + self.assertEqual(package.files_count, 2) + self.assertFalse((Path(self.export_directory.name) / "tmp").exists()) + + def test_prepared_files_include_all_records_and_hide_provider_mentions(self): + organization = Organization.objects.create( + name='ООО "Публичная выгрузка"', + inn="7707083888", + ogrn="1027700132088", + kpp="770701008", + okpo="11223344", + ) + extension = PlannedInspectionExtension.objects.create( + organization=organization, + title="Плановые проверки Генпрокуратуры России", + ) + source_record = OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="checko", + external_id="checko-inspection:123", + title="Запись Checko", + url="https://api.checko.ru/v2/inspections/123", + payload={ + "provider": "Checko", + "provider_alias": "Чеко", + "provider_url": "https://checko.ru/company/123", + "checkout_marker": "checkout_sha", + }, + ) + included_record = OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="official-registry", + external_id="inspection:456", + title="Официальная запись", + payload={"checkout_marker": "checkout_sha"}, + ) + + with TemporaryDirectory() as temporary_directory: + export_directory = Path(temporary_directory) + spool_path = export_directory / "rows.json" + headers, records_count = _spool_source_group_rows( + source_group=SourceGroup.PLANNED_INSPECTIONS.value, + output_path=spool_path, + export_year=timezone.localdate().year, + ) + + self.assertEqual( + headers[:5], + ["Наименование", "ИНН", "ОГРН", "КПП", "ОКПО"], + ) + self.assertEqual(records_count, 2) + spooled_rows = json.loads(spool_path.read_text(encoding="utf-8")) + rows_by_uid = {row["uid"]: row for row in spooled_rows} + self.assertEqual( + set(rows_by_uid), {str(source_record.uid), str(included_record.uid)} + ) + provider_row = rows_by_uid[str(source_record.uid)] + self.assertEqual(provider_row["ОКПО"], "11223344") + self.assertEqual(provider_row["source"], "external-source") + self.assertEqual(provider_row["payload.provider"], "external-source") + self.assertEqual(provider_row["payload.checkout_marker"], "checkout_sha") + + for file_format in ("json", "csv", "xlsx"): + artifact_path = export_directory / f"artifact.{file_format}" + _render_source_group_artifact( + row_spool_path=spool_path, + output_path=artifact_path, + headers=headers, + file_format=file_format, + records_count=records_count, + ) + if file_format == "json": + exported_values = json.loads( + artifact_path.read_text(encoding="utf-8") + ) + elif file_format == "csv": + with artifact_path.open( + encoding="utf-8-sig", + newline="", + ) as csv_file: + exported_values = list(csv.reader(csv_file)) + else: + workbook = load_workbook(artifact_path, read_only=True) + exported_values = list(workbook["data"].iter_rows(values_only=True)) + serialized_values = str(exported_values) + self.assertIn(str(source_record.uid), serialized_values) + self.assertIn(str(included_record.uid), str(exported_values)) + self.assertNotRegex( + serialized_values, + r"(?i)(? Response: + nonlocal calls + calls += 1 + if calls == 1: + return Response(status=504, body=b"gateway timeout") + return Response( + body=b'{"ok":true,"total_count":1,"items":[]}', + headers={"Content-Type": "application/json"}, + ) + + with HTTPTestServer() as server: + server.add_route("POST", "/pp719v2/pub/prod/b/", handle_page) + client = GispProductsClient( + base_url=server.base_url, + max_pages=1, + retry_backoff_seconds=0, + http_adapter=server.adapter, + ) + with self.assertRaisesMessage( + GispProductsClientError, + "empty page before total_count", + ): + client.fetch_products() + + self.assertEqual(calls, 2) + def test_fetch_products_follows_skip_pagination_and_maps_items(self): requests: list[dict] = [] diff --git a/tests/apps/parsers/test_proverki_client.py b/tests/apps/parsers/test_proverki_client.py index 2bd8a65..9829b09 100644 --- a/tests/apps/parsers/test_proverki_client.py +++ b/tests/apps/parsers/test_proverki_client.py @@ -9,6 +9,7 @@ import tempfile import types from asyncio import events as asyncio_events from pathlib import Path +from unittest.mock import patch from xml.etree import ElementPath as element_path from xml.etree import ElementTree as ET @@ -563,6 +564,18 @@ class ProverkiParseXMLTest(SimpleTestCase): inspections = client._parse_xml_content(xml) self.assertEqual(len(inspections), 1) + def test_large_xml_streaming_does_not_build_sanitized_string(self): + xml = _xml_with_tag("INSPECTION", _inspection_attrs()) + client = ProverkiClient() + client.STREAMING_THRESHOLD_BYTES = 1 + with patch.object( + client, + "_sanitize_xml", + side_effect=AssertionError("large XML must stay as bytes"), + ): + inspections = client._parse_xml_content(xml) + self.assertEqual(len(inspections), 1) + def test_parse_xml_record_missing_fields_returns_none(self): element = ET.fromstring("") # noqa: S314 client = ProverkiClient() diff --git a/tests/apps/parsers/test_services.py b/tests/apps/parsers/test_services.py index f4b021b..f1e07e1 100644 --- a/tests/apps/parsers/test_services.py +++ b/tests/apps/parsers/test_services.py @@ -441,8 +441,8 @@ class ParserLoadLogServiceTest(TestCase): self.assertEqual(updated, 0) self.assertEqual(log.status, ParserLoadLog.Status.IN_PROGRESS) - def test_mark_stale_in_progress_failed_closes_precreated_job_without_batch(self): - """Pre-created source-card jobs without batch_id are still linked by source.""" + def test_mark_stale_in_progress_failed_keeps_recent_heartbeat_without_batch(self): + """A recent heartbeat keeps a pre-created source-card job alive.""" log = ParserLoadLogFactory( source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, batch_id=2, @@ -465,9 +465,9 @@ class ParserLoadLogServiceTest(TestCase): log.refresh_from_db() job.refresh_from_db() - self.assertEqual(updated, 1) - self.assertEqual(log.status, ParserLoadLog.Status.FAILED) - self.assertEqual(job.status, JobStatus.FAILURE) + self.assertEqual(updated, 0) + self.assertEqual(log.status, ParserLoadLog.Status.IN_PROGRESS) + self.assertEqual(job.status, JobStatus.STARTED) @unittest.skip( diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index 68fb3d5..1026673 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -392,7 +392,7 @@ class SourceCardServiceUnitTest(SimpleTestCase): ) stale_in_progress_load = SimpleNamespace( status="in_progress", - updated_at=timezone.now() - timedelta(hours=3), + updated_at=timezone.now() - timedelta(hours=5), ) self.assertEqual( SourceCardService._get_status( @@ -618,7 +618,7 @@ class SourceCardServiceDatabaseTest(TestCase): ) self.assertEqual(procurements_card["records_count"], 1) - def test_get_active_tasks_ignores_old_jobs_even_when_updated_recently(self): + def test_get_active_tasks_keeps_old_job_with_recent_heartbeat(self): job = BackgroundJob.objects.create( task_id="old-source-task", task_name="apps.parsers.tasks.parse_industrial_products", @@ -637,7 +637,8 @@ class SourceCardServiceDatabaseTest(TestCase): SourceCardService.get_definition("manufacturers-and-products") ) - self.assertEqual(tasks, []) + self.assertEqual(len(tasks), 1) + self.assertEqual(tasks[0]["task_id"], job.task_id) def test_get_active_tasks_keeps_recent_pending_jobs(self): BackgroundJob.objects.create( diff --git a/tests/apps/parsers/test_source_registry.py b/tests/apps/parsers/test_source_registry.py index fd47794..355875b 100644 --- a/tests/apps/parsers/test_source_registry.py +++ b/tests/apps/parsers/test_source_registry.py @@ -1,3 +1,5 @@ +from dataclasses import asdict + from apps.parsers import tasks from apps.parsers.clients.common.structured import MAX_FILE_SIZE_BYTES from apps.parsers.source_registry import PARSER_SOURCES @@ -40,3 +42,14 @@ class ParserSourceRegistryFNSTest(SimpleTestCase): TASKS_BY_NAME[source.task_name], tasks.sync_fns_financial_reports, ) + + +class ParserSourceRegistryPresentationTest(SimpleTestCase): + def test_public_source_metadata_does_not_name_external_provider(self): + public_metadata = " ".join( + str(value) + for descriptor in PARSER_SOURCES.values() + for value in asdict(descriptor).values() + ) + + self.assertNotIn("checko", public_metadata.lower()) diff --git a/tests/apps/parsers/test_sources_api_e2e.py b/tests/apps/parsers/test_sources_api_e2e.py index cf74476..5fbeb5e 100644 --- a/tests/apps/parsers/test_sources_api_e2e.py +++ b/tests/apps/parsers/test_sources_api_e2e.py @@ -133,6 +133,11 @@ class SourcesApiE2ETest(APITestCase): self.assertEqual(minprom_card["records_count"], 3) self.assertEqual(minprom_card["organizations_count"], 1) self.assertEqual(len(minprom_card["source_items"]), 3) + minprom_items = {item["code"]: item for item in minprom_card["source_items"]} + self.assertEqual( + minprom_items["industrial_products"]["refresh_key"], + "mpt_products", + ) statuses = {item["slug"]: item for item in statuses_response.data["data"]} self.assertEqual(statuses["planned-inspections"]["progress"], 55) diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 1a4e52f..0eb70c2 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -1086,9 +1086,16 @@ class GenericSourceFetchTestCase(TestCase): ) as jobs_cleanup_mock: result = parser_tasks.cleanup_stale_parser_loads(max_age_minutes=45) - cleanup_mock.assert_called_once_with(max_age_minutes=45) + cleanup_mock.assert_called_once_with( + max_age_minutes=45, + pending_max_age_minutes=24 * 60, + ) jobs_cleanup_mock.assert_called_once() self.assertEqual(jobs_cleanup_mock.call_args.kwargs["max_age_minutes"], 45) + self.assertEqual( + jobs_cleanup_mock.call_args.kwargs["pending_max_age_minutes"], + 24 * 60, + ) self.assertIn( "apps.parsers.tasks.scan_fns_directory", jobs_cleanup_mock.call_args.kwargs["task_names"], @@ -1101,6 +1108,7 @@ class GenericSourceFetchTestCase(TestCase): self.assertEqual(result["marked_failed"], 2) self.assertEqual(result["marked_jobs_failed"], 3) self.assertEqual(result["max_age_minutes"], 45) + self.assertEqual(result["pending_max_age_minutes"], 24 * 60) def test_get_or_create_background_job_merges_meta_for_precreated_job(self): BackgroundJobService.create_job( diff --git a/tests/apps/parsers/test_views.py b/tests/apps/parsers/test_views.py index 5d035f4..f3836a0 100644 --- a/tests/apps/parsers/test_views.py +++ b/tests/apps/parsers/test_views.py @@ -11,6 +11,7 @@ import zipfile from datetime import date from unittest.mock import Mock, patch +from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import ( FinancialReport, FinancialReportLine, @@ -81,11 +82,9 @@ def _build_fns_excel_bytes() -> bytes: wb = Workbook() ws = wb.active year = fake.random_int(min=2020, max=2025) - ws.append(["Form", None, year, None]) - ws.append([None, "Code", "Start", "End"]) - ws.append( - [fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)] - ) + ws.append(["Форма №1", None, year, None]) + ws.append([None, "Код", "Начало", "Конец"]) + ws.append(["Баланс", "1600", fake.random_int(10, 999), fake.random_int(10, 999)]) buf = io.BytesIO() wb.save(buf) wb.close() @@ -1413,3 +1412,38 @@ class ParsersViewSetTest(APITestCase): for key, value in payload.items(): self.assertEqual(task_kwargs[key], value) self.assertEqual(task_kwargs["requested_by_id"], self.user.id) + + def test_run_industrial_products_uses_legacy_alias(self): + self.client.force_authenticate(self.user) + url = reverse("api_v1:parsers:run-parser", args=["industrial_products"]) + + with patch( + "apps.parsers.views.tasks.parse_industrial_products.apply_async", + return_value=Mock(id="task-products"), + ) as apply_async_mock: + response = self.client.post(url, {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + apply_async_mock.assert_called_once() + queued_task_id = apply_async_mock.call_args.kwargs["task_id"] + job = BackgroundJob.objects.get(task_id=queued_task_id) + self.assertEqual(job.meta["source_key"], "mpt_products") + + def test_run_parser_reuses_fresh_active_job(self): + self.client.force_authenticate(self.user) + existing = BackgroundJob.objects.create( + task_id="active-products", + task_name="apps.parsers.tasks.parse_industrial_products", + status=JobStatus.STARTED, + ) + url = reverse("api_v1:parsers:run-parser", args=["industrial_products"]) + + with patch( + "apps.parsers.views.tasks.parse_industrial_products.apply_async" + ) as apply_async_mock: + response = self.client.post(url, {}, format="json") + + self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual(response.data["data"]["task_id"], existing.task_id) + self.assertTrue(response.data["data"]["already_running"]) + apply_async_mock.assert_not_called() diff --git a/tests/test_api_inventory_e2e.py b/tests/test_api_inventory_e2e.py index c83a93d..38fc076 100644 --- a/tests/test_api_inventory_e2e.py +++ b/tests/test_api_inventory_e2e.py @@ -3,7 +3,7 @@ from __future__ import annotations import io -from datetime import date, timedelta +from datetime import timedelta from pathlib import Path from tempfile import TemporaryDirectory from types import SimpleNamespace @@ -45,10 +45,10 @@ def _build_fns_excel_bytes() -> bytes: workbook = Workbook() worksheet = workbook.active year = fake.random_int(min=2020, max=2025) - worksheet.append(["Form", None, year, None]) - worksheet.append([None, "Code", "Start", "End"]) + worksheet.append(["Форма №1", None, year, None]) + worksheet.append([None, "Код", "Начало", "Конец"]) worksheet.append( - [fake.word(), _digits(4), fake.random_int(10, 999), fake.random_int(10, 999)] + ["Баланс", "1600", fake.random_int(10, 999), fake.random_int(10, 999)] ) buffer = io.BytesIO() workbook.save(buffer) @@ -634,7 +634,9 @@ class ExchangeApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): ) self.assertEqual(copy_response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) - self.assertEqual(tables_response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED) + self.assertEqual( + tables_response.status_code, status.HTTP_405_METHOD_NOT_ALLOWED + ) self.assertEqual( detail_periodic.status_code, status.HTTP_405_METHOD_NOT_ALLOWED,