From 9b3da5b505ff75c867e4ad0db8ec3c0f4d2a3d65 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 12 Oct 2026 09:47:28 +0200 Subject: [PATCH] fix: implement backend testing amendments and atomic source publication --- docs/backend-testing-amendments-2026-10.md | 121 +++++ .../source-records-public-runtime-contract.md | 42 +- src/apps/core/admin.py | 2 +- src/apps/core/job_presentation.py | 38 ++ .../migrations/0002_canonical_job_statuses.py | 65 +++ src/apps/core/models.py | 30 +- src/apps/core/serializers.py | 65 +-- src/apps/core/services.py | 11 +- src/apps/core/views.py | 11 +- src/apps/parsers/aggregate_publication.py | 214 +++++++++ .../0049_aggregate_refresh_publication.py | 102 +++++ src/apps/parsers/models.py | 44 ++ src/apps/parsers/serializers.py | 46 +- src/apps/parsers/services.py | 20 +- src/apps/parsers/source_cards.py | 129 ++++-- src/apps/parsers/tasks.py | 147 +++--- src/apps/parsers/tasks_registry_snapshots.py | 2 +- src/apps/parsers/views.py | 16 +- src/core/openapi.json | 430 ++++++++++++++++-- .../commands/normalize_testing_amendments.py | 110 +++++ src/organizations/serializers.py | 27 ++ src/organizations/source_backfill.py | 9 +- src/organizations/source_ingestion.py | 28 +- src/organizations/source_record_export.py | 56 ++- .../source_record_normalization.py | 94 ++++ src/organizations/source_record_query.py | 51 +++ src/organizations/source_record_schema.py | 8 + src/organizations/views.py | 141 +++++- tests/apps/core/test_admin.py | 4 +- .../core/test_job_acceptance_lifecycle.py | 107 +++++ tests/apps/core/test_job_control_race.py | 8 +- .../organizations/test_canonical_openapi.py | 19 + .../test_checko_section_contract.py | 2 +- .../test_openapi_runtime_fixtures.py | 26 ++ .../test_source_export_snapshot_contract.py | 2 +- .../test_source_record_export.py | 71 ++- .../organizations/test_testing_amendments.py | 387 ++++++++++++++++ .../parsers/test_aggregate_publication.py | 386 ++++++++++++++++ tests/apps/parsers/test_refresh_progress.py | 26 +- .../parsers/test_snapshot_public_contract.py | 18 +- .../apps/parsers/test_source_cards_service.py | 4 +- tests/apps/parsers/test_sources_api_e2e.py | 7 +- .../apps/parsers/test_system_logs_checkup.py | 8 +- tests/apps/parsers/test_tasks.py | 99 ++-- tools/openapi/build.mjs | 13 + tools/openapi/check-runtime.mjs | 7 + 46 files changed, 2918 insertions(+), 335 deletions(-) create mode 100644 docs/backend-testing-amendments-2026-10.md create mode 100644 src/apps/core/job_presentation.py create mode 100644 src/apps/core/migrations/0002_canonical_job_statuses.py create mode 100644 src/apps/parsers/aggregate_publication.py create mode 100644 src/apps/parsers/migrations/0049_aggregate_refresh_publication.py create mode 100644 src/organizations/management/commands/normalize_testing_amendments.py create mode 100644 src/organizations/source_record_normalization.py create mode 100644 tests/apps/core/test_job_acceptance_lifecycle.py create mode 100644 tests/apps/organizations/test_testing_amendments.py create mode 100644 tests/apps/parsers/test_aggregate_publication.py diff --git a/docs/backend-testing-amendments-2026-10.md b/docs/backend-testing-amendments-2026-10.md new file mode 100644 index 0000000..3c672d0 --- /dev/null +++ b/docs/backend-testing-amendments-2026-10.md @@ -0,0 +1,121 @@ +# Поправки backend после проверки 8 октября + +Основание: файл `backend-testing-8-10-amendments.md`, полученный от Глеба в Telegram +9 октября 2026, сообщение 395926. Этот документ описывает реализацию и её локальную +приёмку; применение к dev/production и нормализация рабочих данных сюда не входят. + +## Записи и фильтры + +- `organization__is_branch`: boolean ordering, false/true/null по возрастанию, + true/false/null по убыванию; uid обеспечивает стабильную пагинацию. +- `payload__issue_date_normalized`, `payload__expiry_date_normalized` для industrial, + `payload__term_date` для ФСТЭК, `payload__valid_from` для лицензий: ISO dates, + null/пустое значение в конце в обоих направлениях. +- Inclusive ranges `first_event_date_from/to` применяются к + `payload.structure.first_event_date`, `license_valid_from_from/to` — к + `payload.valid_from`, `traded_on_from/to` — к `payload.trading.traded_on` обеих бирж. + Неверная дата, обратный диапазон и несовместимая группа дают HTTP 400. +- GET `/api/v2/organization-source-records/filter-options/` принимает + `source_group=leasing|company_messages`, `field=message_type`, optional `search` + до 200 символов. Возвращает уникальные value/label/records_count из той же + видимой таблицы, что list, с сортировкой label/value. Авторизация как у list. +- Числовое ordering amount в закупках и арбитраже сохранено и проверяется отдельно. + +Нормализация выполняется при ingestion и legacy backfill. Для ФСТЭК три raw-ключа +`Срок действия сертификата`, `Срок действия лицензии`, `Срок действия` (в таком +порядке приоритета) дают `term_raw` и nullable ISO `term_date`. Для industrial +нормализуются issue_date/expiry_date. Непарсируемый raw сохраняется, дата становится +null и записывается warning без upstream body. + +Миграционное правило филиалов: известное `Organization.is_branch` имеет приоритет; +затем legacy `Organization.filial`; при отсутствии обоих используется единственное +согласованное значение payload.is_branch/filial. `.T./true/1` и `.F./false/0` +распознаются без учёта регистра. При конфликтах сохраняется каноническое значение +и выдаётся warning. Если неизвестная организация встречается в нескольких старых +записях, первым закрепляется значение при обходе по uid; дальнейшие конфликты +логируются. Payload.is_branch сверяется с организацией, raw filial сохраняется. +Экспорт добавляет `organization.is_branch`, а raw остаётся доступным как raw. + +## Задачи и общая публикация + +Канонические API statuses: queued/running/retrying/success/skipped/failed/cancelled. +Активный progress не достигает 100; terminal событие нельзя заменить поздним +success/failure/retry. Политика skipped: дата последней успешной публикации и +latest_success_load сохраняются; latest_load показывает пропуск. + +Составные карточки `manufacturers-and-products` и `defense-unreliable-suppliers` +создают durable run до отправки первой задачи. Сетевые запросы выполняются вне +транзакции публикации. Подготовленные inputs хранятся приватными чанками по 500 +записей, без записи в публичные таблицы. Последняя часть берёт row locks запуска +и jobs и публикует все части, логи и terminal статусы одной транзакцией. +При любой ошибке публикации транзакция откатывается целиком. Failure/cancellation +одной части закрывает остальные и удаляет staging; поздние workers не публикуют +данные. Повторная доставка уже принятой части не запускает fetch повторно. +Cleanup просроченных jobs использует тот же abort. Worker, потерявший процесс, +остаётся до существующего stale cleanup; возобновление такого запуска требует +нового refresh. Метаданные завершённых запусков сохраняются, inputs удаляются. + +Публичные сообщения jobs/cards/history/CSV используют безопасный фиксированный +текст ошибки; внутренняя диагностика сохраняется в БД и server logs. Поиск по +административному журналу продолжает находить исходный диагностический текст. + +## Экспорт и upload + +Export-ticket по-прежнему принимает JSON. Export-download канонически принимает +`application/x-www-form-urlencoded` с ticket и выдаёт ZIP, attachment filename, +`Cache-Control: no-store`. Старые JSON/multipart вызовы совместимы. Invalid/unknown +400, expired/reused 410, unavailable 503; 503 не расходует capability. Ticket +закрепляет пользователя, sources, format и generation. All-sources filename короткий. + +Действующий upload интегрированных структур уже принимает ZIP до 125 МиБ с XLSX +до 20 МиБ и RTF до 5 МиБ каждый/100 МиБ суммарно, не более 500 RTF. Контракт санкций +РОПК — XLSX до 25 МиБ. Эти лимиты и отрицательные случаи покрыты существующими +upload tests; [инструкция ИС](integrated-structures-import.md) соответствует runtime. +Canonical OpenAPI строится из DRF; ZIP success и JSON errors описаны разными +media types. Orval/TypeScript и runtime Zod fixtures входят в локальную проверку. + +## Применение к рабочей базе + +Сначала резервная копия БД и остановка writers/workers/beat и выдачи новых tickets. +Нельзя смешивать старые workers со схемой канонических статусов. Из каталога backend, +с явным settings окружения, выполнить: + +```sh +python src/manage.py migrate --noinput +python src/manage.py normalize_testing_amendments +python src/manage.py normalize_testing_amendments --apply +python src/manage.py build_source_record_exports +``` + +Первый вызов нормализации — dry run с количествами предполагаемых изменений. +Перед `--apply` проверить warnings и правило конфликтов. Apply не меняет uid, +created_at, updated_at записей и raw; увеличивает revision затронутых published +артефактов и инвалидирует cache. Для legacy sources без artifact revision нет: +выдача tickets должна оставаться остановленной до обязательной пересборки exports. +Ранее выданные capabilities сохраняют pinned generation до своего TTL. +После проверки list/detail/card/job/form download можно возобновить traffic/workers. +Для отката статусов остановить новые workers перед обратной миграцией core; +изменённые payload/филиалы восстанавливаются из резервной копии. + +## Локальная проверка 12 октября 2026 + +- Полный `PYTHONPATH=src uv run --no-sync pytest -q --disable-warnings --tb=short`: + **2045 passed, 46 skipped, 13 warnings**, 85.47 s. Skipped не считаются пройденными. +- Отдельный запуск с `--ds=settings.test_postgres` на временном PostgreSQL 16: + **67 passed**. Включены `test_aggregate_publication.py`, + `test_job_acceptance_lifecycle.py`, `test_testing_amendments.py`, + `test_job_control_race.py` и `test_source_export_snapshot_contract.py`. + Применены реальные миграции; проверены row locks, одновременное завершение частей, + независимый читатель до commit, rollback, cancellation и повторная доставка. + Временный контейнер удалён после проверки. +- Django `check`: ошибок нет. `makemigrations core parsers --check --dry-run` + с включёнными миграциями: изменений моделей без миграции нет. +- Ruff check и format check: прошли для всех 41 изменённых/новых Python-файлов; + `git diff --check` прошёл. +- Swagger и canonical OpenAPI пересозданы, `bun build.mjs --check` прошёл: + 101 path, 192 schemas. Orval generation и `bun run type-check` прошли. + Генерация runtime fixtures отдельным pytest прошла; + `bun run contract-check /absolute/runtime-fixtures.json` проверил **48 ответов API**. + +Это локальные backend/contract проверки на тестовых данных. Выкладка, применение +backfill к рабочей БД и приёмка через рабочий frontend не выполнялись. diff --git a/docs/source-records-public-runtime-contract.md b/docs/source-records-public-runtime-contract.md index 807e79f..f366de5 100644 --- a/docs/source-records-public-runtime-contract.md +++ b/docs/source-records-public-runtime-contract.md @@ -1,4 +1,4 @@ -# Карточки, задачи, история и готовые выгрузки — 14.09.2026 +# Карточки, задачи, история и готовые выгрузки — 12.10.2026 Этот документ уточняет общий runtime-контракт по новому [source-records-backend-improvements.md](source-integration/source-records-backend-improvements.md). @@ -58,10 +58,26 @@ GET `/api/v1/jobs/{task_id}/` для этих трёх источников ра счётчики HTTP/parse ошибок и nullable дата реестра. В публичный meta входят только разрешённые счётчики, без конфигурации loader или raw responses. Те же счётчики доступны в log meta; terminal result имеет aliases `records_count` / `organizations_count`. -Legacy jobs сохраняют прежние значения -статуса и прежний result. Общая схема допускает legacy `error`, deprecated raw aliases -`pending|started|retry|failure|revoked` и дополнительные -поля result. Generic parser results list имеет уникальный operationId +Все jobs возвращают канонические статусы; data migration переводит сохранённые +`pending|started|retry|failure|error|revoked` в `queued|running|retrying|failed|cancelled`. +Приём legacy status в фильтре списка сохранён. `skipped` — terminal результат +задачи с `result.status=skipped`, сохранённый как status=skipped; он не сдвигает дату успешной публикации. +Активный progress ограничен 99; complete устанавливает 100. Карточки и dashboard +показывают `failed` вместо `error`, исключают terminal задачи из active_tasks. +Публичные ошибки jobs, карточек, истории и CSV используют безопасное описание; +исходная диагностика остаётся в базе и серверных логах. + +Составные обновления Минпромторга и поставщиков ГОЗ сохраняют подготовленные +входные данные в приватных таблицах `ParserRefreshRun`/`ParserRefreshStage`. +До готовности всех частей публичные записи и успешная история не меняются; +подготовленные задачи остаются running с progress до 99. Последняя часть одной +транзакцией публикует записи, логи и success/skipped всех частей. Ошибка, отмена +или cleanup просроченной задачи закрывает весь запуск и удаляет staging; старые +записи сохраняются. Повторная доставка не заменяет staging и не открывает terminal +запуск. Те же правила применяются к `parse_all_minpromtorg` и его вызову из +`parse_all_sources`. Отдельные вызовы одиночных парсеров сохраняют прежний scope. + +Generic parser results list имеет уникальный operationId `api_v1_parsers_results_list`; detail сохраняет прежний `api_v1_parsers_results_read`. Dashboard сохраняет operationId `api_v1_parsers_dashboard_list` и tag `api`, чтобы generated client оставался в прежнем модуле. @@ -70,9 +86,13 @@ Dashboard сохраняет operationId `api_v1_parsers_dashboard_list` и tag POST `/api/v2/organization-source-records/export-ticket/` с `sources` и `format` возвращает HTTP 201 `{ticket, file_name, expires_in}`. Ticket одноразовый, -TTL по умолчанию 300 секунд. POST `/export-download/` с form field `ticket` -возвращает ZIP HTTP 200; истёкший, неизвестный или использованный ticket — -HTTP 410 `{detail, code: "ticket_expired"}`. Capability позволяет native form +TTL по умолчанию 300 секунд. В cache ticket закреплены user_id, sources, format и +immutable generation; download использует ticket как bearer capability. POST `/export-download/` с form field `ticket` +возвращает ZIP HTTP 200 с `Cache-Control: no-store`; неверный или неизвестный +ticket — HTTP 400 `{detail, code: "invalid_ticket"}`, истёкший или использованный — +HTTP 410 `{detail, code: "ticket_expired"}`. Маркер выпуска сохраняется ещё сутки +после TTL для различения неизвестного и истёкшего ticket. Временная недоступность +архива даёт 503 и не расходует ticket. Capability позволяет native form без передачи JWT в URL. Для выпуска ticket нужны прежние admin permissions. Публикация любого из трёх снимков запрашивает существующую фоновую сборку @@ -101,3 +121,9 @@ payload требуется запустить существующую кома Локальные контрактные тесты не подтверждают живой upstream СРО или принятие frontend UI. Доступ к upstream по умолчанию выключен; включение требует отдельного согласованного основания. Настоящие адреса/секреты стендов в эти проверки не входят. + + +## Поправки проверки от 8 октября + +Реализованный query, правила нормализации, проверки upload и порядок применения: +[backend-testing-amendments-2026-10.md](backend-testing-amendments-2026-10.md). diff --git a/src/apps/core/admin.py b/src/apps/core/admin.py index 7fe9e2e..c22b028 100644 --- a/src/apps/core/admin.py +++ b/src/apps/core/admin.py @@ -145,7 +145,7 @@ class BackgroundJobAdmin(admin.ModelAdmin): from celery import current_app count = 0 - for job in queryset.filter(status__in=["pending", "started"]): + for job in queryset.filter(status__in=["queued", "running", "retrying"]): current_app.control.revoke(job.task_id, terminate=True) job.revoke() count += 1 diff --git a/src/apps/core/job_presentation.py b/src/apps/core/job_presentation.py new file mode 100644 index 0000000..bd049c7 --- /dev/null +++ b/src/apps/core/job_presentation.py @@ -0,0 +1,38 @@ +"""Public job state and error vocabulary, independent of Celery transport states.""" + +LEGACY_JOB_STATUSES = { + "pending": "queued", + "started": "running", + "retry": "retrying", + "failure": "failed", + "error": "failed", + "revoked": "cancelled", +} +CANONICAL_JOB_STATUSES = ( + "queued", + "running", + "retrying", + "success", + "skipped", + "failed", + "cancelled", +) + + +def job_status(job): + status = LEGACY_JOB_STATUSES.get(job.status, job.status) + if ( + status == "success" + and isinstance(job.result, dict) + and job.result.get("status") == "skipped" + ): + return "skipped" + return status + + +def public_error_message(value): + # Diagnostics remain in the stored job/log and server logs. Never guess + # whether a free-form upstream exception is safe to display. + if value == "Задача отозвана пользователем": + return value + return "Не удалось обновить данные. Повторите попытку позже." if value else "" diff --git a/src/apps/core/migrations/0002_canonical_job_statuses.py b/src/apps/core/migrations/0002_canonical_job_statuses.py new file mode 100644 index 0000000..24b100f --- /dev/null +++ b/src/apps/core/migrations/0002_canonical_job_statuses.py @@ -0,0 +1,65 @@ +# Generated by Django 3.2.25 on 2026-10-12 07:16 + +from django.db import migrations, models + +STATUS_MAP = { + "pending": "queued", + "started": "running", + "retry": "retrying", + "failure": "failed", + "error": "failed", + "revoked": "cancelled", +} + + +def forward_statuses(apps, schema_editor): + jobs = apps.get_model("core", "BackgroundJob").objects.using( + schema_editor.connection.alias + ) + for old, new in STATUS_MAP.items(): + jobs.filter(status=old).update(status=new) + jobs.filter(status="success", result__status="skipped").update(status="skipped") + + +def reverse_statuses(apps, schema_editor): + jobs = apps.get_model("core", "BackgroundJob").objects.using( + schema_editor.connection.alias + ) + for new, old in { + "skipped": "success", + "queued": "pending", + "running": "started", + "retrying": "retry", + "failed": "failure", + "cancelled": "revoked", + }.items(): + jobs.filter(status=new).update(status=old) + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0001_background_job"), + ] + + operations = [ + migrations.RunPython(forward_statuses, reverse_statuses), + migrations.AlterField( + model_name="backgroundjob", + name="status", + field=models.CharField( + choices=[ + ("queued", "Ожидает"), + ("running", "Выполняется"), + ("success", "Успешно"), + ("skipped", "Пропущена"), + ("failed", "Ошибка"), + ("cancelled", "Отменена"), + ("retrying", "Повтор"), + ], + db_index=True, + default="queued", + max_length=20, + verbose_name="статус", + ), + ), + ] diff --git a/src/apps/core/models.py b/src/apps/core/models.py index bacb9bc..6d6bcde 100644 --- a/src/apps/core/models.py +++ b/src/apps/core/models.py @@ -20,12 +20,13 @@ from django.utils.translation import gettext_lazy as _ class JobStatus(models.TextChoices): """Статусы фоновых задач.""" - PENDING = "pending", _("Ожидает") - STARTED = "started", _("Выполняется") + PENDING = "queued", _("Ожидает") + STARTED = "running", _("Выполняется") SUCCESS = "success", _("Успешно") - FAILURE = "failure", _("Ошибка") - REVOKED = "revoked", _("Отменена") - RETRY = "retry", _("Повтор") + SKIPPED = "skipped", _("Пропущена") + FAILURE = "failed", _("Ошибка") + REVOKED = "cancelled", _("Отменена") + RETRY = "retrying", _("Повтор") class BackgroundJob(TimestampMixin, models.Model): @@ -164,7 +165,7 @@ class BackgroundJob(TimestampMixin, models.Model): self, progress: int, message: str = "", *, meta: dict | None = None ) -> None: """Атомарно сохранить только неубывающий прогресс незавершённой задачи.""" - progress = min(max(int(progress), 0), 100) + progress = min(max(int(progress), 0), 99) values = { "progress": progress, "progress_message": message, @@ -178,7 +179,9 @@ class BackgroundJob(TimestampMixin, models.Model): def complete(self, result: Any = None) -> None: """Первое финальное событие фиксирует успешный результат и 100%.""" self._update_unfinished( - status=JobStatus.SUCCESS, + status=JobStatus.SKIPPED + if isinstance(result, dict) and result.get("status") == "skipped" + else JobStatus.SUCCESS, progress=100, result=result, error="", @@ -188,6 +191,11 @@ class BackgroundJob(TimestampMixin, models.Model): def fail(self, error: str, traceback_str: str = "") -> None: """Зафиксировать ошибку без изменения достигнутого прогресса.""" + if self.meta.get("aggregate_refresh_id"): + from apps.parsers.aggregate_publication import abort_for_job + + if abort_for_job(self, error, traceback_str=traceback_str): + return self._update_unfinished( status=JobStatus.FAILURE, error=str(error), @@ -197,6 +205,11 @@ class BackgroundJob(TimestampMixin, models.Model): def revoke(self, message: str | None = None) -> None: """Отменить незавершённую задачу, сохранив её прогресс.""" + if self.meta.get("aggregate_refresh_id"): + from apps.parsers.aggregate_publication import abort_for_job + + if abort_for_job(self, message or "Обновление отменено", cancelled=True): + return values = {"status": JobStatus.REVOKED, "completed_at": timezone.now()} if message is not None: values["progress_message"] = message @@ -213,6 +226,7 @@ class BackgroundJob(TimestampMixin, models.Model): """Проверка, завершена ли задача.""" return self.status in ( JobStatus.SUCCESS, + JobStatus.SKIPPED, JobStatus.FAILURE, JobStatus.REVOKED, ) @@ -220,7 +234,7 @@ class BackgroundJob(TimestampMixin, models.Model): @property def is_successful(self) -> bool: """Проверка успешного завершения.""" - return self.status == JobStatus.SUCCESS + return self.status in (JobStatus.SUCCESS, JobStatus.SKIPPED) @property def duration(self) -> float | None: diff --git a/src/apps/core/serializers.py b/src/apps/core/serializers.py index 746e0af..a1d2201 100644 --- a/src/apps/core/serializers.py +++ b/src/apps/core/serializers.py @@ -5,6 +5,11 @@ - BackgroundJob - статус фоновых задач """ +from apps.core.job_presentation import ( + CANONICAL_JOB_STATUSES, + job_status, + public_error_message, +) from drf_yasg.utils import swagger_serializer_method from rest_framework import serializers @@ -13,34 +18,7 @@ SNAPSHOT_SOURCES = ( "fns_sme_support_recipients", "sro_membership_check", ) -JOB_API_STATUSES = ( - "queued", - "running", - "retrying", - "success", - "failed", - "cancelled", - "skipped", - "error", - # Deprecated transport aliases retained for existing generated clients. - "pending", - "started", - "retry", - "failure", - "revoked", -) - - -def _get_frontend_job_status(raw_status: str) -> str: - mapping = { - "pending": "running", - "started": "running", - "retry": "running", - "success": "success", - "failure": "error", - "revoked": "error", - } - return mapping.get(raw_status, raw_status) +JOB_API_STATUSES = CANONICAL_JOB_STATUSES def _get_job_source(job) -> str | None: @@ -55,22 +33,7 @@ def _get_job_source(job) -> str | None: def _get_job_status(job) -> str: - if _get_job_source(job) is None: - return _get_frontend_job_status(job.status) - if ( - job.status == "success" - and isinstance(job.result, dict) - and job.result.get("status") == "skipped" - ): - return "skipped" - return { - "pending": "queued", - "started": "running", - "retry": "retrying", - "success": "success", - "failure": "failed", - "revoked": "cancelled", - }.get(job.status, job.status) + return job_status(job) class SnapshotRunMetadataSerializer(serializers.Serializer): @@ -136,7 +99,7 @@ class BackgroundJobSerializer(serializers.Serializer): task_id = serializers.CharField(read_only=True) status = serializers.SerializerMethodField() - progress = serializers.IntegerField(read_only=True) + progress = serializers.IntegerField(read_only=True, min_value=0, max_value=100) message = serializers.SerializerMethodField() result = serializers.SerializerMethodField() meta = serializers.SerializerMethodField() @@ -161,7 +124,7 @@ class BackgroundJobSerializer(serializers.Serializer): serializer_or_field=SnapshotJobResultSerializer(allow_null=True) ) def get_result(self, obj): - return obj.result + return None if job_status(obj) in {"failed", "cancelled"} else obj.result @swagger_serializer_method( serializer_or_field=serializers.ChoiceField( @@ -175,18 +138,20 @@ class BackgroundJobSerializer(serializers.Serializer): serializer_or_field=serializers.CharField(allow_null=True) ) def get_message(self, obj): + if job_status(obj) in {"failed", "cancelled"}: + return public_error_message(obj.error or obj.progress_message) or None return obj.progress_message or None @swagger_serializer_method( serializer_or_field=serializers.CharField(allow_null=True) ) def get_error(self, obj): - return obj.error or None + return public_error_message(obj.error) or None @swagger_serializer_method( serializer_or_field=serializers.ChoiceField( choices=JOB_API_STATUSES, - help_text="Registry jobs use queued/running/retrying/success/failed/cancelled/skipped. Raw pending/started/retry/failure/revoked are deprecated legacy aliases.", + help_text="All jobs use queued/running/retrying/success/failed/cancelled/skipped. Legacy stored values are normalized. Skipped does not advance the last successful publication date.", ) ) def get_status(self, obj): @@ -201,7 +166,7 @@ class BackgroundJobListSerializer(serializers.Serializer): task_id = serializers.CharField(read_only=True) name = serializers.SerializerMethodField() status = serializers.SerializerMethodField() - progress = serializers.IntegerField(read_only=True) + progress = serializers.IntegerField(read_only=True, min_value=0, max_value=100) source = serializers.SerializerMethodField() def to_representation(self, instance): @@ -224,7 +189,7 @@ class BackgroundJobListSerializer(serializers.Serializer): @swagger_serializer_method( serializer_or_field=serializers.ChoiceField( choices=JOB_API_STATUSES, - help_text="Registry jobs use queued/running/retrying/success/failed/cancelled/skipped. Raw pending/started/retry/failure/revoked are deprecated legacy aliases.", + help_text="All jobs use queued/running/retrying/success/failed/cancelled/skipped. Legacy stored values are normalized. Skipped does not advance the last successful publication date.", ) ) def get_status(self, obj): diff --git a/src/apps/core/services.py b/src/apps/core/services.py index 54c5821..2e38bc1 100644 --- a/src/apps/core/services.py +++ b/src/apps/core/services.py @@ -700,7 +700,9 @@ class BackgroundJobService(BaseReadOnlyService): """ qs = cls.get_queryset().filter(user_id=user_id) if status: - qs = qs.filter(status=status) + from apps.core.job_presentation import LEGACY_JOB_STATUSES + + qs = qs.filter(status=LEGACY_JOB_STATUSES.get(status, status)) return qs[:limit] @classmethod @@ -740,7 +742,12 @@ class BackgroundJobService(BaseReadOnlyService): deleted, _ = ( cls.get_queryset() .filter( - status__in=[JobStatus.SUCCESS, JobStatus.FAILURE, JobStatus.REVOKED], + status__in=[ + JobStatus.SUCCESS, + JobStatus.SKIPPED, + JobStatus.FAILURE, + JobStatus.REVOKED, + ], completed_at__lt=cutoff, ) .delete() diff --git a/src/apps/core/views.py b/src/apps/core/views.py index 386ac86..1d99c74 100644 --- a/src/apps/core/views.py +++ b/src/apps/core/views.py @@ -288,13 +288,14 @@ class BackgroundJobStreamView(BackgroundJobStatusView): } def _build_final_payload(self, job) -> tuple[str, dict[str, Any]]: + presentation = BackgroundJobSerializer(job).data if job.is_successful: return ( "completed", { "task_id": job.task_id, - "status": "success", - "progress": job.progress, + "status": presentation["status"], + "progress": presentation["progress"], "result": job.result, }, ) @@ -303,11 +304,9 @@ class BackgroundJobStreamView(BackgroundJobStatusView): "failed", { "task_id": job.task_id, - "status": "error", + "status": presentation["status"], "progress": job.progress, - "message": job.error - or job.progress_message - or "Задача завершилась с ошибкой", + "message": presentation["error"] or presentation["message"], }, ) diff --git a/src/apps/parsers/aggregate_publication.py b/src/apps/parsers/aggregate_publication.py new file mode 100644 index 0000000..8cea3f2 --- /dev/null +++ b/src/apps/parsers/aggregate_publication.py @@ -0,0 +1,214 @@ +"""Stage composed refreshes and publish their records, logs and jobs atomically.""" + +import logging +from dataclasses import asdict, replace +from decimal import Decimal + +from apps.core.models import BackgroundJob, JobStatus +from apps.parsers.models import ParserLoadLog, ParserRefreshRun, ParserRefreshStage +from django.db import transaction +from django.utils import timezone +from organizations.source_cache import invalidate_source_data_cache + +logger = logging.getLogger(__name__) + +ACTIVE = (JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY) + + +def stage_records(run_id, source, load_batch, records): + """Persist only private inputs while the run is collecting.""" + with transaction.atomic(): + run = ParserRefreshRun.objects.select_for_update().get(pk=run_id) + if run.status != "collecting" or source not in run.tasks: + raise RuntimeError("aggregate_refresh_not_collecting") + job = BackgroundJob.objects.get(task_id=run.tasks[source]) + if job.status not in ACTIVE: + raise RuntimeError("aggregate_refresh_part_not_active") + if source in run.ready: + return sum( + len(chunk.records) for chunk in run.staged_chunks.filter(source=source) + ) + run.staged_chunks.filter(source=source).delete() + for offset in range(0, len(records), 500): + chunk = [ + asdict(replace(record, financial_lines=tuple(record.financial_lines))) + for record in records[offset : offset + 500] + ] + ParserRefreshStage.objects.create( + run=run, + source=source, + position=offset // 500, + load_batch=load_batch, + records=chunk, + ) + return len(records) + + +def _deserialize_record(payload): + from organizations.source_ingestion import ( + SourceFinancialLineInput, + SourceRecordInput, + ) + + payload = dict(payload) + if payload.get("amount") is not None: + payload["amount"] = Decimal(str(payload["amount"])) + payload["financial_lines"] = tuple( + SourceFinancialLineInput(**line) for line in payload.get("financial_lines", []) + ) + return SourceRecordInput(**payload) + + +def _publish(run, jobs): + from apps.parsers.services import ParserLoadLogService + from organizations.source_ingestion import OrganizationSourceIngestionService + + for source, task_id in run.tasks.items(): + part = run.ready[source] + saved = 0 + for chunk in ( + run.staged_chunks.filter(source=source) + .order_by("position") + .iterator(chunk_size=1) + ): + result = OrganizationSourceIngestionService.save_records( + source=source, + load_batch=chunk.load_batch, + records=[_deserialize_record(record) for record in chunk.records], + ) + saved += result.created_records + result.updated_records + result = {**part["result"], "saved": saved} + load = ParserLoadLog.objects.get(pk=part["load_id"]) + ParserLoadLogService.update(load, status=result["status"], records_count=saved) + jobs[task_id].complete(result=result) + run.status = "published" + run.save(update_fields=["status", "updated_at"]) + run.staged_chunks.all().delete() + transaction.on_commit(_after_publication) + + +def _after_publication(): + from apps.parsers.tasks_registry_snapshots import _refresh_published_exports + + try: + invalidate_source_data_cache() + except Exception: + # Publication is committed; keep terminal success and still enqueue exports. + logger.exception( + "Aggregate publication committed but cache invalidation failed" + ) + _refresh_published_exports() + + +def finish_part(job, load_log, result): + """Keep prepared parts active until the last part commits the whole run.""" + run_id = job.meta.get("aggregate_refresh_id") + if not run_id: + return None + try: + with transaction.atomic(): + run = ParserRefreshRun.objects.select_for_update().get(pk=run_id) + if run.status == "published": + job.refresh_from_db() + return job.result + if ( + run.status != "collecting" + or run.tasks.get(load_log.source) != job.task_id + ): + raise RuntimeError("aggregate_refresh_not_collecting") + jobs = { + item.task_id: item + for item in BackgroundJob.objects.select_for_update() + .filter(task_id__in=run.tasks.values()) + .order_by("pk") + } + if len(jobs) != len(run.tasks) or any( + item.status not in ACTIVE for item in jobs.values() + ): + raise RuntimeError("aggregate_refresh_part_not_active") + if result["status"] not in {"success", "skipped"}: + raise RuntimeError("aggregate_refresh_part_failed") + run.ready = { + **run.ready, + load_log.source: {"load_id": load_log.pk, "result": result}, + } + run.save(update_fields=["ready", "updated_at"]) + job.update_progress(99, "Данные подготовлены; ожидается общая публикация") + if len(run.ready) == len(run.tasks): + _publish(run, jobs) + job.refresh_from_db() + return job.result + return {**result, "status": "running", "publication_pending": True} + except Exception: + abort_for_job(job, "Не удалось опубликовать составной источник") + raise + + +def abort_for_job(job, error, *, traceback_str="", cancelled=False): + """Close all parts before a failed/cancelled run can publish anything.""" + run_id = (job.meta or {}).get("aggregate_refresh_id") + if not run_id: + return False + with transaction.atomic(): + run = ParserRefreshRun.objects.select_for_update().get(pk=run_id) + if run.status != "collecting": + job.refresh_from_db() + return True + run.status = "failed" + run.save(update_fields=["status", "updated_at"]) + jobs = list( + BackgroundJob.objects.select_for_update() + .filter(task_id__in=run.tasks.values()) + .order_by("pk") + ) + now = timezone.now() + for part in jobs: + if part.status not in ACTIVE: + continue + BackgroundJob.objects.filter(pk=part.pk, status__in=ACTIVE).update( + status=JobStatus.REVOKED if cancelled else JobStatus.FAILURE, + error=str(error), + traceback=traceback_str if part.pk == job.pk else "", + completed_at=now, + updated_at=now, + ) + batch_id = (part.meta or {}).get("batch_id") + ParserLoadLog.objects.filter( + source=part.meta.get("source"), batch_id=batch_id, status="in_progress" + ).update(status="failed", error_message=str(error), updated_at=now) + # Parts marked ready can have a log even when a caller has no batch meta. + ParserLoadLog.objects.filter( + pk__in=[part["load_id"] for part in run.ready.values()], + status="in_progress", + ).update(status="failed", error_message=str(error), updated_at=now) + run.staged_chunks.all().delete() + transaction.on_commit(invalidate_source_data_cache) + job.refresh_from_db() + return True + + +def claim_part(task_id): + """Claim one delivery before fetching; duplicate deliveries cannot replace staging.""" + if not task_id: + return None + job = BackgroundJob.objects.filter(task_id=task_id).first() + if job is None or not job.meta.get("aggregate_refresh_id"): + return None + with transaction.atomic(): + run = ParserRefreshRun.objects.select_for_update().get( + pk=job.meta["aggregate_refresh_id"] + ) + job = BackgroundJob.objects.select_for_update().get(pk=job.pk) + if run.tasks.get(job.meta.get("source")) != task_id: + raise RuntimeError("aggregate_refresh_part_mismatch") + if run.status != "collecting" or job.status not in ACTIVE: + return job.result or {"status": job.status} + if job.meta.get("aggregate_claimed"): + return { + "status": "running", + "publication_pending": True, + "duplicate_delivery": True, + } + job.meta = {**job.meta, "aggregate_claimed": True} + job.save(update_fields=["meta", "updated_at"]) + return None diff --git a/src/apps/parsers/migrations/0049_aggregate_refresh_publication.py b/src/apps/parsers/migrations/0049_aggregate_refresh_publication.py new file mode 100644 index 0000000..5e1d631 --- /dev/null +++ b/src/apps/parsers/migrations/0049_aggregate_refresh_publication.py @@ -0,0 +1,102 @@ +# Generated by Django 3.2.25 on 2026-10-12 07:31 + +import uuid + +import django.core.serializers.json +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0048_enable_requested_source_schedules"), + ] + + operations = [ + migrations.CreateModel( + name="ParserRefreshRun", + fields=[ + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Дата и время создания записи", + verbose_name="создано", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="Дата и время последнего обновления", + verbose_name="обновлено", + ), + ), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("source_card", models.CharField(max_length=100)), + ( + "status", + models.CharField( + choices=[ + ("collecting", "Collecting"), + ("published", "Published"), + ("failed", "Failed"), + ], + default="collecting", + max_length=20, + ), + ), + ("tasks", models.JSONField(default=dict)), + ("ready", models.JSONField(default=dict)), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="ParserRefreshStage", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("source", models.CharField(max_length=100)), + ("position", models.PositiveIntegerField()), + ("load_batch", models.PositiveIntegerField()), + ( + "records", + models.JSONField( + encoder=django.core.serializers.json.DjangoJSONEncoder + ), + ), + ( + "run", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="staged_chunks", + to="parsers.parserrefreshrun", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="parserrefreshstage", + constraint=models.UniqueConstraint( + fields=("run", "source", "position"), name="unique_refresh_stage_chunk" + ), + ), + ] diff --git a/src/apps/parsers/models.py b/src/apps/parsers/models.py index 26ed1f9..841ec13 100644 --- a/src/apps/parsers/models.py +++ b/src/apps/parsers/models.py @@ -9,6 +9,7 @@ import uuid from apps.core.mixins import TimestampMixin from apps.parsers.artifact_storage import ParserArtifactStorage from django.conf import settings +from django.core.serializers.json import DjangoJSONEncoder from django.db import models from django.utils.translation import gettext_lazy as _ @@ -1447,3 +1448,46 @@ class IntegratedStructureEventDocument(models.Model): def __str__(self) -> str: return f"{self.event_id}:{self.document_id}" + + +class ParserRefreshRun(TimestampMixin, models.Model): + """Publication barrier for the explicitly associated tasks of a source card.""" + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + source_card = models.CharField(max_length=100) + status = models.CharField( + max_length=20, + default="collecting", + choices=[ + ("collecting", "Collecting"), + ("published", "Published"), + ("failed", "Failed"), + ], + ) + tasks = models.JSONField(default=dict) + ready = models.JSONField(default=dict) + + def __str__(self): + return f"{self.source_card}:{self.pk}:{self.status}" + + +class ParserRefreshStage(models.Model): + """Private normalized input chunks; never queried by public source APIs.""" + + run = models.ForeignKey( + ParserRefreshRun, on_delete=models.CASCADE, related_name="staged_chunks" + ) + source = models.CharField(max_length=100) + position = models.PositiveIntegerField() + load_batch = models.PositiveIntegerField() + records = models.JSONField(encoder=DjangoJSONEncoder) + + def __str__(self): + return f"{self.run_id}:{self.source}:{self.position}" + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["run", "source", "position"], name="unique_refresh_stage_chunk" + ) + ] diff --git a/src/apps/parsers/serializers.py b/src/apps/parsers/serializers.py index 1a2e76a..b142c17 100644 --- a/src/apps/parsers/serializers.py +++ b/src/apps/parsers/serializers.py @@ -5,6 +5,7 @@ import socket from urllib.parse import urlsplit from apps.core.exceptions import ValidationError as APIValidationError +from apps.core.job_presentation import CANONICAL_JOB_STATUSES, public_error_message from apps.core.serializers import ( BackgroundJobListSerializer, SnapshotRunMetadataSerializer, @@ -793,6 +794,11 @@ class ParserLoadLogSerializer(serializers.ModelSerializer): Информация о каждой загрузке данных из внешнего источника. """ + error_message = serializers.SerializerMethodField() + + def get_error_message(self, obj): + return public_error_message(obj.error_message) + source_display = serializers.CharField(source="get_source_display", read_only=True) source_label = serializers.CharField(source="get_source_display", read_only=True) snapshot = serializers.SerializerMethodField() @@ -1063,8 +1069,8 @@ class SourceCardTaskSerializer(serializers.Serializer): task_id = serializers.CharField(read_only=True) task_name = serializers.CharField(read_only=True) - status = serializers.CharField(read_only=True) - progress = serializers.IntegerField(read_only=True) + status = serializers.ChoiceField(choices=CANONICAL_JOB_STATUSES, read_only=True) + progress = serializers.IntegerField(read_only=True, min_value=0, max_value=100) progress_message = serializers.CharField(read_only=True) started_at = serializers.DateTimeField(read_only=True, allow_null=True) created_at = serializers.DateTimeField(read_only=True) @@ -1096,12 +1102,27 @@ class SourceCardSerializer(serializers.Serializer): description = serializers.CharField(read_only=True) order = serializers.IntegerField(read_only=True) is_available = serializers.BooleanField(read_only=True) - status = serializers.CharField(read_only=True) + status = serializers.ChoiceField( + choices=( + "idle", + "unavailable", + "pending", + "in_progress", + "success", + "skipped", + "failed", + ), + read_only=True, + ) status_label = serializers.CharField(read_only=True) - progress = serializers.IntegerField(read_only=True) + progress = serializers.IntegerField(read_only=True, min_value=0, max_value=100) records_count = serializers.IntegerField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) - last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True) + last_updated_at = serializers.DateTimeField( + read_only=True, + allow_null=True, + help_text="Дата успешной публикации. Skipped и failed не сдвигают дату; составной источник публикуется после готовности всех частей.", + ) next_update_at = serializers.DateTimeField(read_only=True, allow_null=True) error_message = serializers.CharField(read_only=True) task_names = serializers.ListField(child=serializers.CharField(), read_only=True) @@ -1122,13 +1143,24 @@ class SourceTaskStatusSerializer(serializers.Serializer): row_number = serializers.IntegerField(read_only=True) slug = serializers.CharField(read_only=True) source = serializers.CharField(read_only=True) - status = serializers.CharField(read_only=True) + status = serializers.ChoiceField( + choices=( + "idle", + "unavailable", + "pending", + "in_progress", + "success", + "skipped", + "failed", + ), + read_only=True, + ) status_label = serializers.CharField(read_only=True) actualized_at = serializers.DateTimeField(read_only=True, allow_null=True) next_update_at = serializers.DateTimeField(read_only=True, allow_null=True) records_count = serializers.IntegerField(read_only=True) organizations_count = serializers.IntegerField(read_only=True) - progress = serializers.IntegerField(read_only=True) + progress = serializers.IntegerField(read_only=True, min_value=0, max_value=100) error_message = serializers.CharField(read_only=True) active_tasks = SourceCardTaskSerializer(many=True, read_only=True) diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index 0434992..6b46f59 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -495,6 +495,7 @@ class IndustrialCertificateService( batch_id: int, *, chunk_size: int = 500, + publication_run_id: str | None = None, ) -> int: """ Сохранить список сертификатов из парсера. @@ -557,8 +558,11 @@ class IndustrialCertificateService( source=ParserLoadLog.Source.INDUSTRIAL, load_batch=batch_id, records=source_records, + publication_run_id=publication_run_id, + ) + saved_count = ( + result.created_records + result.updated_records + result.staged_records ) - saved_count = result.created_records + result.updated_records logger.info("Saved %d certificates", saved_count) return saved_count @@ -608,6 +612,7 @@ class ManufacturerService(BulkOperationsMixin, BaseService[ManufacturerRecord]): batch_id: int, *, chunk_size: int = 500, + publication_run_id: str | None = None, ) -> int: """ Сохранить список производителей из парсера. @@ -653,10 +658,11 @@ class ManufacturerService(BulkOperationsMixin, BaseService[ManufacturerRecord]): source=ParserLoadLog.Source.MANUFACTURES, load_batch=batch_id, records=source_records, + publication_run_id=publication_run_id, ) created_count = result.created_records updated_count = result.updated_records - saved_count = created_count + updated_count + saved_count = created_count + updated_count + result.staged_records logger.info( "Saved %d manufacturers (created=%d, updated=%d)", saved_count, @@ -719,6 +725,7 @@ class IndustrialProductService( batch_id: int, *, chunk_size: int = 1000, + publication_run_id: str | None = None, ) -> int: """Сохранить список промышленной продукции из парсера.""" if not products: @@ -768,10 +775,11 @@ class IndustrialProductService( source=ParserLoadLog.Source.INDUSTRIAL_PRODUCTS, load_batch=batch_id, records=source_records, + publication_run_id=publication_run_id, ) created_count = result.created_records updated_count = result.updated_records - saved_count = created_count + updated_count + saved_count = created_count + updated_count + result.staged_records logger.info( "Saved %d industrial products (created=%d, updated=%d)", saved_count, @@ -886,6 +894,7 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR *, source: str, chunk_size: int = 500, + publication_run_id: str | None = None, ) -> int: """Сохранить нормализованные записи нового источника.""" if not records: @@ -944,8 +953,11 @@ class GenericParserRecordService(BulkOperationsMixin, BaseService[GenericParserR source=record_source, load_batch=batch_id, records=source_records, + publication_run_id=publication_run_id, + ) + saved_count += ( + result.created_records + result.updated_records + result.staged_records ) - saved_count += result.created_records + result.updated_records return saved_count diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index efd4d0a..c2d9aae 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -12,12 +12,14 @@ from typing import Any from apps.core.exceptions import ConflictError from apps.core.exceptions import ValidationError as APIValidationError +from apps.core.job_presentation import job_status, public_error_message from apps.core.models import JobStatus from apps.core.services import BackgroundJobService from apps.parsers.models import ( VACANCY_RECORD_SOURCES, ParserBatchSequence, ParserLoadLog, + ParserRefreshRun, ) from apps.parsers.snapshot_metadata import ( attach_snapshot_artifacts, @@ -39,7 +41,7 @@ from organizations.source_cache import ( from organizations.source_groups import get_source_group_descriptor from rest_framework.exceptions import ValidationError -SUCCESSFUL_LOAD_STATUSES = {"success", "skipped"} +SUCCESSFUL_LOAD_STATUSES = {"success"} ACTIVE_JOB_STATUSES = [JobStatus.PENDING, JobStatus.STARTED, JobStatus.RETRY] STALE_ACTIVE_MAX_AGE_MINUTES = 4 * 60 STALE_PENDING_MAX_AGE_MINUTES = 24 * 60 @@ -770,11 +772,15 @@ class SourceCardService: if refresh_tasks and all( task["status"] not in ACTIVE_JOB_STATUSES for task in refresh_tasks ): - if all(task["status"] == JobStatus.SUCCESS for task in refresh_tasks): - status = "success" + if all(task["status"] in {"success", "skipped"} for task in refresh_tasks): + status = ( + "skipped" + if all(task["status"] == "skipped" for task in refresh_tasks) + else "success" + ) error_message = "" else: - status = "error" + status = "failed" error_message = next( ( task.get("error", "") @@ -794,12 +800,12 @@ class SourceCardService: "upload_url": definition.upload_url, "status": status, "status_label": cls._get_status_label(status), - "progress": progress, + "progress": 100 if status in {"success", "skipped"} else min(99, progress), "records_count": records_count, "organizations_count": organizations_count, "last_updated_at": last_updated_at, "next_update_at": cls._get_next_update_at(definition, last_updated_at), - "error_message": error_message, + "error_message": public_error_message(error_message), "task_names": list(definition.task_names), "refresh_requires_params": any( param.required for param in definition.refresh_params @@ -1558,43 +1564,9 @@ class SourceCardService: ) if existing: return [{"task_id": existing.task_id, "task_name": existing.task_name}] - run_id = str(uuid.uuid4()) - task_ids = [str(uuid.uuid4()) for _ in task_specs] - jobs = [] - with transaction.atomic(): - if definition.slug in SNAPSHOT_CARD_SLUGS.values(): - source = task_specs[0][2] - sequence, _ = ParserBatchSequence.objects.get_or_create(source=source) - ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk) - if ( - BackgroundJobService.get_queryset() - .filter( - task_name=task_specs[0][1], - status__in=ACTIVE_JOB_STATUSES, - ) - .exists() - ): - raise ConflictError( - message="Источник уже обновляется", - code="refresh_already_running", - ) - for task_id, (_, task_name, source) in zip( - task_ids, task_specs, strict=False - ): - meta = { - "source_card": definition.slug, - "source": str(source), - "refresh_run_id": run_id, - "refresh_task_ids": task_ids, - } - jobs.append( - BackgroundJobService.create_job( - task_id=task_id, - task_name=task_name, - user_id=requested_by_id, - meta=meta, - ) - ) + jobs = cls._create_refresh_jobs( + definition, task_specs, requested_by_id=requested_by_id + ) tasks = [] for index, (job, (task, task_name, _source)) in enumerate( zip(jobs, task_specs, strict=False) @@ -1616,6 +1588,61 @@ class SourceCardService: raise return tasks + @classmethod + def _create_refresh_jobs(cls, definition, task_specs, *, requested_by_id): + """Bind every part to one durable publication run before broker dispatch.""" + if len({str(spec[2]) for spec in task_specs}) != len(task_specs): + raise ValueError("Refresh group must contain distinct sources") + run_id = str(uuid.uuid4()) + task_ids = [str(uuid.uuid4()) for _ in task_specs] + jobs = [] + with transaction.atomic(): + if len(task_specs) > 1 or definition.slug in SNAPSHOT_CARD_SLUGS.values(): + source = task_specs[0][2] + sequence, _ = ParserBatchSequence.objects.get_or_create(source=source) + ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk) + if ( + BackgroundJobService.get_queryset() + .filter( + task_name__in=[spec[1] for spec in task_specs], + status__in=ACTIVE_JOB_STATUSES, + ) + .exists() + ): + raise ConflictError( + message="Источник уже обновляется", + code="refresh_already_running", + ) + aggregate_run = None + if len(task_specs) > 1: + aggregate_run = ParserRefreshRun.objects.create( + source_card=definition.slug, + tasks={ + str(spec[2]): task_id + for spec, task_id in zip(task_specs, task_ids, strict=True) + }, + ) + for task_id, (_, task_name, source) in zip( + task_ids, task_specs, strict=False + ): + meta = { + "source_card": definition.slug, + "source": str(source), + "refresh_run_id": run_id, + "refresh_task_ids": task_ids, + } + if aggregate_run is not None: + meta["aggregate_refresh_id"] = str(aggregate_run.pk) + jobs.append( + BackgroundJobService.create_job( + task_id=task_id, + task_name=task_name, + user_id=requested_by_id, + meta=meta, + ) + ) + return jobs + @staticmethod def _get_simple_refresh_task_specs( slug: str, @@ -1950,9 +1977,11 @@ class SourceCardService: ): return "in_progress" if latest_load and latest_load.status == "in_progress": - return "error" + return "failed" if latest_load and latest_load.status == "failed": - return "error" + return "failed" + if latest_load and latest_load.status == "skipped": + return "skipped" if last_updated_at: return "success" return "idle" @@ -2012,6 +2041,8 @@ class SourceCardService: "success": "Обновлено", "in_progress": "В процессе", "error": "Ошибка", + "failed": "Ошибка", + "skipped": "Пропущено", "idle": "Нет данных", "unavailable": "Не подключено", } @@ -2039,9 +2070,11 @@ class SourceCardService: return { "task_id": job.task_id, "task_name": job.task_name, - "status": job.status, + "status": job_status(job), "progress": job.progress, - "progress_message": job.progress_message, + "progress_message": public_error_message(job.error or job.progress_message) + if job_status(job) in {"failed", "cancelled"} + else job.progress_message, "started_at": job.started_at, "created_at": job.created_at, "meta": job.meta, @@ -2068,7 +2101,7 @@ class SourceCardService: else 0, "records_count": load_log.records_count, "status": load_log.status, - "error_message": load_log.error_message, + "error_message": public_error_message(load_log.error_message), "created_at": load_log.created_at, "updated_at": load_log.updated_at, } diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 7009c28..2e4de48 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -19,6 +19,7 @@ from pathlib import Path from apps.core.models import BackgroundJob, JobStatus from apps.core.services import BackgroundJobService from apps.core.tasks import PeriodicTask as CorePeriodicTask +from apps.parsers.aggregate_publication import claim_part, finish_part from apps.parsers.checko_collection import ( claim_monthly_collection, collection_period_month, @@ -318,6 +319,9 @@ def _run_generic_parser( requested_by_id: int | None = None, ) -> dict: """Единый runner для новых разнородных источников.""" + previous_result = claim_part(self.request.id) + if previous_result is not None: + return previous_result load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=source, status="in_progress", @@ -342,7 +346,15 @@ def _run_generic_parser( records, batch_id=batch_id, source=source, + publication_run_id=job.meta.get("aggregate_refresh_id"), ) + prepared = finish_part( + job, + load_log, + {"batch_id": batch_id, "saved": saved_count, "status": "success"}, + ) + if prepared is not None: + return prepared ParserLoadLogService.update( load_log, status="success", @@ -354,6 +366,11 @@ def _run_generic_parser( except ParserSourceSkipped as e: message = str(e) logger.warning("%s skipped: %s", task_name, message) + prepared = finish_part( + job, load_log, {"batch_id": batch_id, "saved": 0, "status": "skipped"} + ) + if prepared is not None: + return prepared ParserLoadLogService.update( load_log, status=ParserLoadLog.Status.SKIPPED, @@ -2084,6 +2101,9 @@ def parse_industrial_production( Результат: batch_id, saved, status """ source = ParserLoadLog.Source.INDUSTRIAL + previous_result = claim_part(self.request.id) + if previous_result is not None: + return previous_result load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=source, status="in_progress", @@ -2126,9 +2146,17 @@ def parse_industrial_production( saved_count = IndustrialCertificateService.save_certificates( certificates, batch_id=batch_id, + publication_run_id=job.meta.get("aggregate_refresh_id"), ) # Обновляем лог + prepared = finish_part( + job, + load_log, + {"batch_id": batch_id, "saved": saved_count, "status": "success"}, + ) + if prepared is not None: + return prepared ParserLoadLogService.update( load_log, status="success", @@ -2176,6 +2204,9 @@ def parse_manufactures( Результат: batch_id, saved, status """ source = ParserLoadLog.Source.MANUFACTURES + previous_result = claim_part(self.request.id) + if previous_result is not None: + return previous_result load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=source, status="in_progress", @@ -2218,9 +2249,17 @@ def parse_manufactures( saved_count = ManufacturerService.save_manufacturers( manufacturers, batch_id=batch_id, + publication_run_id=job.meta.get("aggregate_refresh_id"), ) # Обновляем лог + prepared = finish_part( + job, + load_log, + {"batch_id": batch_id, "saved": saved_count, "status": "success"}, + ) + if prepared is not None: + return prepared ParserLoadLogService.update( load_log, status="success", @@ -2277,6 +2316,9 @@ def parse_industrial_products( Результат: batch_id, saved, status """ source = ParserLoadLog.Source.INDUSTRIAL_PRODUCTS + previous_result = claim_part(self.request.id) + if previous_result is not None: + return previous_result load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=source, status="in_progress", @@ -2320,12 +2362,7 @@ def parse_industrial_products( saved_count = IndustrialProductService.save_products( products, batch_id=batch_id, - ) - - ParserLoadLogService.update( - load_log, - status="success", - records_count=saved_count, + publication_run_id=job.meta.get("aggregate_refresh_id"), ) result = { @@ -2336,6 +2373,15 @@ def parse_industrial_products( "total": fetch_result.total_count, "truncated": fetch_result.truncated, } + prepared = finish_part(job, load_log, result) + if prepared is not None: + return prepared + ParserLoadLogService.update( + load_log, + status="success", + records_count=saved_count, + ) + job.complete(result=result) logger.info( @@ -2374,40 +2420,39 @@ def parse_all_minpromtorg( Returns: Результаты всех парсеров """ - logger.info("Starting all Minpromtorg parsers") + from apps.parsers.source_cards import SourceCardService - if client_adapter is not None: - industrial_result = parse_industrial_production.apply( - kwargs={"proxies": proxies, "client_adapter": client_adapter} - ) - industrial_products_result = parse_industrial_products.apply( - kwargs={"proxies": proxies, "client_adapter": client_adapter} - ) - manufactures_result = parse_manufactures.apply( - kwargs={"proxies": proxies, "client_adapter": client_adapter} - ) - else: - industrial_result = parse_industrial_production.delay( - proxies=proxies, - client_adapter=client_adapter, - ) - industrial_products_result = parse_industrial_products.delay( - proxies=proxies, - client_adapter=client_adapter, - ) - manufactures_result = parse_manufactures.delay( - proxies=proxies, - client_adapter=client_adapter, - ) - - results = { - "industrial": industrial_result.id, - "industrial_products": industrial_products_result.id, - "manufactures": manufactures_result.id, + definition = SourceCardService.get_definition("manufacturers-and-products") + specs = ( + ( + parse_industrial_production, + "apps.parsers.tasks.parse_industrial_production", + "industrial", + ), + ( + parse_industrial_products, + "apps.parsers.tasks.parse_industrial_products", + "industrial_products", + ), + (parse_manufactures, "apps.parsers.tasks.parse_manufactures", "manufactures"), + ) + jobs = SourceCardService._create_refresh_jobs( + definition, specs, requested_by_id=None + ) + try: + for job, (task, _, _) in zip(jobs, specs, strict=True): + kwargs = {"proxies": proxies, "client_adapter": client_adapter} + if client_adapter is not None: + task.apply(kwargs=kwargs, task_id=job.task_id, throw=True) + else: + task.apply_async(kwargs=kwargs, task_id=job.task_id) + except Exception: + jobs[0].fail("Не удалось выполнить составное обновление Минпромторга") + raise + return { + source: job.task_id for job, (_, _, source) in zip(jobs, specs, strict=True) } - return results - @shared_task(bind=True) def parse_inspections( @@ -2545,16 +2590,10 @@ def parse_all_sources( logger.info("Starting all parsers from all sources") from django.conf import settings + minprom_results = parse_all_minpromtorg( + proxies=proxies, client_adapter=client_adapter + ) if client_adapter is not None: - industrial_result = parse_industrial_production.apply( - kwargs={"proxies": proxies, "client_adapter": client_adapter} - ) - industrial_products_result = parse_industrial_products.apply( - kwargs={"proxies": proxies, "client_adapter": client_adapter} - ) - manufactures_result = parse_manufactures.apply( - kwargs={"proxies": proxies, "client_adapter": client_adapter} - ) inspections_result = sync_inspections.apply( kwargs={ "proxies": proxies, @@ -2564,18 +2603,6 @@ def parse_all_sources( ) generic_results = {} else: - industrial_result = parse_industrial_production.delay( - proxies=proxies, - client_adapter=client_adapter, - ) - industrial_products_result = parse_industrial_products.delay( - proxies=proxies, - client_adapter=client_adapter, - ) - manufactures_result = parse_manufactures.delay( - proxies=proxies, - client_adapter=client_adapter, - ) inspections_result = sync_inspections.delay( proxies=proxies, client_adapter=client_adapter, @@ -2594,9 +2621,7 @@ def parse_all_sources( } results = { - "industrial": industrial_result.id, - "industrial_products": industrial_products_result.id, - "manufactures": manufactures_result.id, + **minprom_results, "inspections": inspections_result.id, **generic_results, } diff --git a/src/apps/parsers/tasks_registry_snapshots.py b/src/apps/parsers/tasks_registry_snapshots.py index e805656..d5099fd 100644 --- a/src/apps/parsers/tasks_registry_snapshots.py +++ b/src/apps/parsers/tasks_registry_snapshots.py @@ -258,7 +258,7 @@ def _run_snapshot( except Exception as exc: if job is not None: job.refresh_from_db() - if job.status == JobStatus.SUCCESS: + if job.is_successful: return job.result # Store a bounded error class, not HTTP request internals or raw source data. message = _snapshot_error_message(exc) diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index dd2f42e..e57b5ac 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -12,6 +12,7 @@ from collections import defaultdict from datetime import date, datetime, time, timedelta from apps.core.filters import BaseFilterSet +from apps.core.job_presentation import public_error_message from apps.core.models import JobStatus from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag from apps.core.response import api_error_response, api_response @@ -690,7 +691,7 @@ def _serialize_parser_log_row(log: ParserLoadLog) -> dict: ), "status": log.status, "status_label": _get_parser_log_status_label(log.status), - "error_message": log.error_message, + "error_message": public_error_message(log.error_message), "created_at": log.created_at, "updated_at": log.updated_at, } @@ -810,7 +811,13 @@ def _filtered_parser_log_rows(request, queryset) -> list[dict]: rows = [_serialize_parser_log_row(log) for log in queryset] search_term = request.query_params.get("search", "").strip() if search_term: - rows = [row for row in rows if _matches_parser_log_search(row, search_term)] + rows = [ + row + for log, row in zip(queryset, rows, strict=False) + if _matches_parser_log_search( + {**row, "error_message": log.error_message}, search_term + ) + ] ordering = request.query_params.get("ordering", "") if any( field.lstrip("-") == "organizations_count" @@ -2385,7 +2392,10 @@ def _registry_enrichment_analytics( ) recent_other = max(len(latest_loads) - recent_success - recent_failed, 0) running_jobs = sum( - 1 for job in jobs if job.status in {"pending", "started", "retry"} + 1 + for job in jobs + if job.status + in {"queued", "running", "retrying", "pending", "started", "retry"} ) return { diff --git a/src/core/openapi.json b/src/core/openapi.json index 9562093..056da6e 100644 --- a/src/core/openapi.json +++ b/src/core/openapi.json @@ -9258,6 +9258,26 @@ "type": "string" } }, + { + "name": "first_event_date_from", + "in": "query", + "description": "Фильтр записей источника: first_event_date_from.", + "required": false, + "schema": { + "type": "string", + "format": "date" + } + }, + { + "name": "first_event_date_to", + "in": "query", + "description": "Фильтр записей источника: first_event_date_to.", + "required": false, + "schema": { + "type": "string", + "format": "date" + } + }, { "name": "has_bankruptcy_information", "in": "query", @@ -9359,6 +9379,26 @@ "type": "string" } }, + { + "name": "license_valid_from_from", + "in": "query", + "description": "Фильтр записей источника: license_valid_from_from.", + "required": false, + "schema": { + "type": "string", + "format": "date" + } + }, + { + "name": "license_valid_from_to", + "in": "query", + "description": "Фильтр записей источника: license_valid_from_to.", + "required": false, + "schema": { + "type": "string", + "format": "date" + } + }, { "name": "license_valid_until_from", "in": "query", @@ -9616,6 +9656,26 @@ "format": "date" } }, + { + "name": "traded_on_from", + "in": "query", + "description": "Фильтр записей источника: traded_on_from.", + "required": false, + "schema": { + "type": "string", + "format": "date" + } + }, + { + "name": "traded_on_to", + "in": "query", + "description": "Фильтр записей источника: traded_on_to.", + "required": false, + "schema": { + "type": "string", + "format": "date" + } + }, { "name": "has_registry", "in": "query", @@ -9727,7 +9787,7 @@ { "name": "ordering", "in": "query", - "description": "Сортировка по полям: record_date, record_type, external_id, organization__name, extension__organization__name, extension__organization__full_name, created_at, updated_at, title, uid, extension__organization__inn, extension__organization__ogrn, extension__organization__okpo, status, payload__attestation_status, payload__full_name, payload__medo_address, payload__registration_number, payload__uk_hm_treasury, payload__european_union, payload__united_states, payload__switzerland, payload__united_states_sectoral, payload__uk_uksl, payload__ukraine, payload__sentiment, payload__news_source, events_count, documents_count, first_event_date, payload__exchange, payload__instrument_id, payload__instrument_kind, payload__listing_date, payload__trading__traded_on, payload__publication_date, payload__message_type, payload__license_number, payload__issued_on, payload__valid_until, payload__authority, payload__document_raw, payload__terminated_on, payload__service_text, organization__full_name, organization__inn, organization__ogrn, organization__okpo, payload__classification__organization_type__name, payload__classification__establishment_kind__name, payload__budget__level__name, payload__address__region__name, payload__is_separate_division, payload__support_form__name, payload__support_kind__name, payload__support_until, payload__provider__name, payload__sme_category__name, payload__has_violation, amount, payload__membership_status, payload__region, payload__sro_name, payload__admission_date, payload__status__name, payload__executive_body_name, payload__registration_date, payload__publication_count, payload__reorganization_count, payload__captured_at. Для обратной сортировки используйте префикс -. Несколько полей перечисляются через запятую без пробелов, например -record_date,-external_id. Значения record_date с null сортируются последними.", + "description": "Сортировка по полям: record_date, record_type, external_id, organization__name, extension__organization__name, extension__organization__full_name, created_at, updated_at, title, uid, extension__organization__inn, extension__organization__ogrn, extension__organization__okpo, status, payload__attestation_status, payload__full_name, payload__medo_address, payload__registration_number, payload__uk_hm_treasury, payload__european_union, payload__united_states, payload__switzerland, payload__united_states_sectoral, payload__uk_uksl, payload__ukraine, payload__sentiment, payload__news_source, events_count, documents_count, first_event_date, organization__is_branch, payload__issue_date_normalized, payload__expiry_date_normalized, payload__term_date, payload__valid_from, payload__exchange, payload__instrument_id, payload__instrument_kind, payload__listing_date, payload__trading__traded_on, payload__publication_date, payload__message_type, payload__license_number, payload__issued_on, payload__valid_until, payload__authority, payload__document_raw, payload__terminated_on, payload__service_text, organization__full_name, organization__inn, organization__ogrn, organization__okpo, payload__classification__organization_type__name, payload__classification__establishment_kind__name, payload__budget__level__name, payload__address__region__name, payload__is_separate_division, payload__support_form__name, payload__support_kind__name, payload__support_until, payload__provider__name, payload__sme_category__name, payload__has_violation, amount, payload__membership_status, payload__region, payload__sro_name, payload__admission_date, payload__status__name, payload__executive_body_name, payload__registration_date, payload__publication_count, payload__reorganization_count, payload__captured_at. Для обратной сортировки используйте префикс -. Несколько полей перечисляются через запятую без пробелов, например -record_date,-external_id. Значения record_date с null сортируются последними.", "required": false, "schema": { "anyOf": [ @@ -9776,9 +9836,17 @@ "description": "Read-only flat API for source records across source extensions.", "requestBody": { "content": { - "application/json": { + "application/x-www-form-urlencoded": { "schema": { - "$ref": "#/components/schemas/OrganizationSourceRecordExportDownload" + "type": "object", + "properties": { + "ticket": { + "type": "string" + } + }, + "required": [ + "ticket" + ] } } }, @@ -9788,7 +9856,7 @@ "200": { "description": "Потоковый ZIP-архив записей источников.", "content": { - "application/json": { + "application/zip": { "schema": { "type": "string", "format": "binary" @@ -9797,7 +9865,14 @@ } }, "400": { - "description": "Ticket отсутствует или имеет неверный формат." + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SourceRecordExportInvalidTicket" + } + } + } }, "410": { "description": "", @@ -9880,7 +9955,7 @@ "200": { "description": "ZIP-архив записей источников.", "content": { - "application/json": { + "application/zip": { "schema": { "type": "string", "format": "binary" @@ -9911,6 +9986,92 @@ }, "parameters": [] }, + "/api/v2/organization-source-records/filter-options/": { + "get": { + "operationId": "v2_organization_source_records_filter_options", + "description": "Read-only flat API for source records across source extensions.", + "parameters": [ + { + "name": "page", + "in": "query", + "description": "A page number within the paginated result set.", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "page_size", + "in": "query", + "description": "Number of results to return per page.", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "source_group", + "in": "query", + "description": "Группа опубликованных записей", + "required": true, + "schema": { + "type": "string", + "enum": [ + "leasing", + "company_messages" + ] + } + }, + { + "name": "field", + "in": "query", + "description": "Поле фильтра", + "required": true, + "schema": { + "type": "string", + "enum": [ + "message_type" + ] + } + }, + { + "name": "search", + "in": "query", + "description": "Поиск по коду или названию, до 200 символов", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SourceRecordFilterOptionsResponse" + } + } + } + }, + "400": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SourceRecordErrorResponse" + } + } + } + } + }, + "tags": [ + "Организации" + ] + }, + "parameters": [] + }, "/api/v2/organization-source-records/{uid}/": { "get": { "operationId": "api_v2_organization_source_records_read", @@ -12010,29 +12171,25 @@ }, "status": { "title": "Status", - "description": "Registry jobs use queued/running/retrying/success/failed/cancelled/skipped. Raw pending/started/retry/failure/revoked are deprecated legacy aliases.", + "description": "All jobs use queued/running/retrying/success/failed/cancelled/skipped. Legacy stored values are normalized. Skipped does not advance the last successful publication date.", "type": "string", "enum": [ "queued", "running", "retrying", "success", - "failed", - "cancelled", "skipped", - "error", - "pending", - "started", - "retry", - "failure", - "revoked" + "failed", + "cancelled" ], "readOnly": true }, "progress": { "title": "Progress", "type": "integer", - "readOnly": true + "readOnly": true, + "maximum": 100, + "minimum": 0 }, "source": { "title": "Source", @@ -12380,29 +12537,25 @@ }, "status": { "title": "Status", - "description": "Registry jobs use queued/running/retrying/success/failed/cancelled/skipped. Raw pending/started/retry/failure/revoked are deprecated legacy aliases.", + "description": "All jobs use queued/running/retrying/success/failed/cancelled/skipped. Legacy stored values are normalized. Skipped does not advance the last successful publication date.", "type": "string", "enum": [ "queued", "running", "retrying", "success", - "failed", - "cancelled", "skipped", - "error", - "pending", - "started", - "retry", - "failure", - "revoked" + "failed", + "cancelled" ], "readOnly": true }, "progress": { "title": "Progress", "type": "integer", - "readOnly": true + "readOnly": true, + "maximum": 100, + "minimum": 0 }, "message": { "title": "Message", @@ -13129,11 +13282,9 @@ "readOnly": true }, "error_message": { - "title": "Сообщение об ошибке", - "description": "Текст ошибки при неудачной загрузке", + "title": "Error message", "type": "string", - "readOnly": true, - "minLength": 1 + "readOnly": true }, "created_at": { "title": "Создано", @@ -13822,13 +13973,23 @@ "status": { "title": "Status", "type": "string", - "readOnly": true, - "minLength": 1 + "enum": [ + "queued", + "running", + "retrying", + "success", + "skipped", + "failed", + "cancelled" + ], + "readOnly": true }, "progress": { "title": "Progress", "type": "integer", - "readOnly": true + "readOnly": true, + "maximum": 100, + "minimum": 0 }, "progress_message": { "title": "Progress message", @@ -14065,8 +14226,16 @@ "status": { "title": "Status", "type": "string", - "readOnly": true, - "minLength": 1 + "enum": [ + "idle", + "unavailable", + "pending", + "in_progress", + "success", + "skipped", + "failed" + ], + "readOnly": true }, "status_label": { "title": "Status label", @@ -14077,7 +14246,9 @@ "progress": { "title": "Progress", "type": "integer", - "readOnly": true + "readOnly": true, + "maximum": 100, + "minimum": 0 }, "records_count": { "title": "Records count", @@ -14091,6 +14262,7 @@ }, "last_updated_at": { "title": "Last updated at", + "description": "Дата успешной публикации. Skipped и failed не сдвигают дату; составной источник публикуется после готовности всех частей.", "type": "string", "format": "date-time", "readOnly": true, @@ -14213,8 +14385,16 @@ "status": { "title": "Status", "type": "string", - "readOnly": true, - "minLength": 1 + "enum": [ + "idle", + "unavailable", + "pending", + "in_progress", + "success", + "skipped", + "failed" + ], + "readOnly": true }, "status_label": { "title": "Status label", @@ -14249,7 +14429,9 @@ "progress": { "title": "Progress", "type": "integer", - "readOnly": true + "readOnly": true, + "maximum": 100, + "minimum": 0 }, "error_message": { "title": "Error message", @@ -14329,8 +14511,16 @@ "status": { "title": "Status", "type": "string", - "readOnly": true, - "minLength": 1 + "enum": [ + "idle", + "unavailable", + "pending", + "in_progress", + "success", + "skipped", + "failed" + ], + "readOnly": true }, "status_label": { "title": "Status label", @@ -14341,7 +14531,9 @@ "progress": { "title": "Progress", "type": "integer", - "readOnly": true + "readOnly": true, + "maximum": 100, + "minimum": 0 }, "records_count": { "title": "Records count", @@ -14355,6 +14547,7 @@ }, "last_updated_at": { "title": "Last updated at", + "description": "Дата успешной публикации. Skipped и failed не сдвигают дату; составной источник публикуется после готовности всех частей.", "type": "string", "format": "date-time", "readOnly": true, @@ -17781,6 +17974,34 @@ "OrganizationSourceRecordPayload": { "type": "object", "properties": { + "issue_date_normalized": { + "title": "Issue date normalized", + "type": "string", + "format": "date", + "readOnly": true, + "nullable": true + }, + "expiry_date_normalized": { + "title": "Expiry date normalized", + "type": "string", + "format": "date", + "readOnly": true, + "nullable": true + }, + "term_raw": { + "title": "Term raw", + "type": "string", + "readOnly": true, + "minLength": 1, + "nullable": true + }, + "term_date": { + "title": "Term date", + "type": "string", + "format": "date", + "readOnly": true, + "nullable": true + }, "artifact_id": { "title": "Artifact id", "type": "string", @@ -18515,17 +18736,24 @@ } } }, - "OrganizationSourceRecordExportDownload": { + "SourceRecordExportInvalidTicket": { "required": [ - "ticket" + "detail", + "code" ], "type": "object", "properties": { - "ticket": { - "title": "Ticket", + "detail": { + "title": "Detail", "type": "string", - "maxLength": 64, "minLength": 1 + }, + "code": { + "title": "Code", + "type": "string", + "enum": [ + "invalid_ticket" + ] } } }, @@ -18645,9 +18873,105 @@ } } }, + "SourceRecordFilterOption": { + "required": [ + "value", + "label", + "records_count" + ], + "type": "object", + "properties": { + "value": { + "title": "Value", + "type": "string", + "minLength": 1 + }, + "label": { + "title": "Label", + "type": "string", + "minLength": 1 + }, + "records_count": { + "title": "Records count", + "type": "integer", + "minimum": 0 + } + } + }, + "SourceRecordFilterOptionsMeta": { + "required": [ + "total_count" + ], + "type": "object", + "properties": { + "total_count": { + "title": "Total count", + "type": "integer", + "minimum": 0 + } + } + }, + "SourceRecordFilterOptionsResponse": { + "required": [ + "success", + "data", + "errors", + "meta" + ], + "type": "object", + "properties": { + "success": { + "title": "Success", + "type": "boolean" + }, + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SourceRecordFilterOption" + } + }, + "errors": { + "type": "array", + "items": { + "$ref": "#/components/schemas/SourceRecordError" + } + }, + "meta": { + "$ref": "#/components/schemas/SourceRecordFilterOptionsMeta" + } + } + }, "OrganizationSourceRecordDetailPayload": { "type": "object", "properties": { + "issue_date_normalized": { + "title": "Issue date normalized", + "type": "string", + "format": "date", + "readOnly": true, + "nullable": true + }, + "expiry_date_normalized": { + "title": "Expiry date normalized", + "type": "string", + "format": "date", + "readOnly": true, + "nullable": true + }, + "term_raw": { + "title": "Term raw", + "type": "string", + "readOnly": true, + "minLength": 1, + "nullable": true + }, + "term_date": { + "title": "Term date", + "type": "string", + "format": "date", + "readOnly": true, + "nullable": true + }, "artifact_id": { "title": "Artifact id", "type": "string", @@ -21106,6 +21430,16 @@ "-documents_count", "first_event_date", "-first_event_date", + "organization__is_branch", + "-organization__is_branch", + "payload__issue_date_normalized", + "-payload__issue_date_normalized", + "payload__expiry_date_normalized", + "-payload__expiry_date_normalized", + "payload__term_date", + "-payload__term_date", + "payload__valid_from", + "-payload__valid_from", "payload__exchange", "-payload__exchange", "payload__instrument_id", @@ -21190,7 +21524,7 @@ }, "SourceRecordOrderingSequence": { "type": "string", - "pattern": "^(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|payload__exchange|-payload__exchange|payload__instrument_id|-payload__instrument_id|payload__instrument_kind|-payload__instrument_kind|payload__listing_date|-payload__listing_date|payload__trading__traded_on|-payload__trading__traded_on|payload__publication_date|-payload__publication_date|payload__message_type|-payload__message_type|payload__license_number|-payload__license_number|payload__issued_on|-payload__issued_on|payload__valid_until|-payload__valid_until|payload__authority|-payload__authority|payload__document_raw|-payload__document_raw|payload__terminated_on|-payload__terminated_on|payload__service_text|-payload__service_text|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at)(?:,(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|payload__exchange|-payload__exchange|payload__instrument_id|-payload__instrument_id|payload__instrument_kind|-payload__instrument_kind|payload__listing_date|-payload__listing_date|payload__trading__traded_on|-payload__trading__traded_on|payload__publication_date|-payload__publication_date|payload__message_type|-payload__message_type|payload__license_number|-payload__license_number|payload__issued_on|-payload__issued_on|payload__valid_until|-payload__valid_until|payload__authority|-payload__authority|payload__document_raw|-payload__document_raw|payload__terminated_on|-payload__terminated_on|payload__service_text|-payload__service_text|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at))+$", + "pattern": "^(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|organization__is_branch|-organization__is_branch|payload__issue_date_normalized|-payload__issue_date_normalized|payload__expiry_date_normalized|-payload__expiry_date_normalized|payload__term_date|-payload__term_date|payload__valid_from|-payload__valid_from|payload__exchange|-payload__exchange|payload__instrument_id|-payload__instrument_id|payload__instrument_kind|-payload__instrument_kind|payload__listing_date|-payload__listing_date|payload__trading__traded_on|-payload__trading__traded_on|payload__publication_date|-payload__publication_date|payload__message_type|-payload__message_type|payload__license_number|-payload__license_number|payload__issued_on|-payload__issued_on|payload__valid_until|-payload__valid_until|payload__authority|-payload__authority|payload__document_raw|-payload__document_raw|payload__terminated_on|-payload__terminated_on|payload__service_text|-payload__service_text|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at)(?:,(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|organization__is_branch|-organization__is_branch|payload__issue_date_normalized|-payload__issue_date_normalized|payload__expiry_date_normalized|-payload__expiry_date_normalized|payload__term_date|-payload__term_date|payload__valid_from|-payload__valid_from|payload__exchange|-payload__exchange|payload__instrument_id|-payload__instrument_id|payload__instrument_kind|-payload__instrument_kind|payload__listing_date|-payload__listing_date|payload__trading__traded_on|-payload__trading__traded_on|payload__publication_date|-payload__publication_date|payload__message_type|-payload__message_type|payload__license_number|-payload__license_number|payload__issued_on|-payload__issued_on|payload__valid_until|-payload__valid_until|payload__authority|-payload__authority|payload__document_raw|-payload__document_raw|payload__terminated_on|-payload__terminated_on|payload__service_text|-payload__service_text|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at))+$", "description": "Two or more allowlisted ordering fields, separated by commas without spaces." }, "AttachmentUnavailableError": { diff --git a/src/organizations/management/commands/normalize_testing_amendments.py b/src/organizations/management/commands/normalize_testing_amendments.py new file mode 100644 index 0000000..aaa4f8b --- /dev/null +++ b/src/organizations/management/commands/normalize_testing_amendments.py @@ -0,0 +1,110 @@ +"""Backfill normalized dates and branch flags without reloading upstream data.""" + +import json + +from apps.parsers.models import ParserSourceArtifact +from django.core.management.base import BaseCommand +from django.db import transaction + +from organizations.models import Organization, OrganizationSourceRecord +from organizations.source_cache import invalidate_source_data_cache +from organizations.source_record_normalization import ( + BRANCH_GROUPS, + canonical_branch, + normalize_record_payload, +) + + +class Command(BaseCommand): + help = "Backfill 8 October acceptance fields; dry run unless --apply is supplied" + + def add_arguments(self, parser): + parser.add_argument("--apply", action="store_true") + + def handle(self, *args, **options): + apply = options["apply"] + result = { + "dry_run": not apply, + "scanned": 0, + "changed_records": 0, + "changed_organizations": 0, + } + sources = set() + simulated_branches = {} + with transaction.atomic(): + queryset = OrganizationSourceRecord.objects.filter( + extension__source_group__in=BRANCH_GROUPS + ).order_by("uid") + if apply: + queryset = queryset.select_for_update() + # Keyset batches keep memory bounded and avoid server-side cursor writes. + last_uid = None + while True: + page = ( + queryset if last_uid is None else queryset.filter(uid__gt=last_uid) + ) + records = list( + page.select_related("extension", "extension__organization")[:500] + ) + if not records: + break + for record in records: + self._normalize_record( + record, apply, result, sources, simulated_branches + ) + last_uid = records[-1].uid + if apply and sources: + for artifact in ParserSourceArtifact.objects.select_for_update().filter( + source__in=sources, status=ParserSourceArtifact.Status.PUBLISHED + ): + metadata = dict(artifact.metadata) + metadata["normalized_payload_revision"] = ( + int(metadata.get("normalized_payload_revision", 0)) + 1 + ) + artifact.metadata = metadata + artifact.save(update_fields=["metadata"]) + invalidate_source_data_cache() + transaction.on_commit(invalidate_source_data_cache) + result["exports_rebuild_required"] = bool(apply and sources) + return json.dumps(result, sort_keys=True) + + @staticmethod + def _normalize_record(record, apply, result, sources, simulated_branches): + result["scanned"] += 1 + organization = record.extension.organization + if apply: + organization = Organization.objects.select_for_update().get( + pk=organization.pk + ) + elif organization.pk in simulated_branches: + organization.is_branch = simulated_branches[organization.pk] + previous = organization.is_branch + payload = normalize_record_payload( + record.source, + record.payload, + organization=organization, + source_group=record.extension.source_group, + persist_branch=apply, + ) + if not apply and previous is None: + organization.is_branch = canonical_branch( + organization, payload, source=record.source + ) + if organization.is_branch is not None: + simulated_branches[organization.pk] = organization.is_branch + if organization.is_branch != previous: + result["changed_organizations"] += 1 + sources.update( + OrganizationSourceRecord.objects.filter( + extension__organization=organization + ) + .values_list("source", flat=True) + .distinct() + ) + if payload != record.payload: + result["changed_records"] += 1 + sources.add(record.source) + if apply: + OrganizationSourceRecord.objects.filter(pk=record.pk).update( + payload=payload + ) diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index 48da52b..8ca9e74 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -103,6 +103,11 @@ class SourceSupportProviderSerializer(serializers.Serializer): class OrganizationSourceRecordPayloadSerializer(serializers.Serializer): """Typed optional fields exposed by source-record list payloads.""" + issue_date_normalized = serializers.DateField(read_only=True, allow_null=True) + expiry_date_normalized = serializers.DateField(read_only=True, allow_null=True) + term_raw = serializers.CharField(read_only=True, allow_null=True) + term_date = serializers.DateField(read_only=True, allow_null=True) + artifact_id = serializers.UUIDField(read_only=True, allow_null=True) support_registry_number = serializers.CharField(read_only=True) recipient_type = serializers.CharField(read_only=True) @@ -554,6 +559,23 @@ class SourceRecordErrorResponseSerializer(serializers.Serializer): meta = serializers.JSONField(allow_null=True) +class SourceRecordFilterOptionSerializer(serializers.Serializer): + value = serializers.CharField() + label = serializers.CharField() + records_count = serializers.IntegerField(min_value=0) + + +class SourceRecordFilterOptionsMetaSerializer(serializers.Serializer): + total_count = serializers.IntegerField(min_value=0) + + +class SourceRecordFilterOptionsResponseSerializer(serializers.Serializer): + success = serializers.BooleanField() + data = SourceRecordFilterOptionSerializer(many=True) + errors = SourceRecordErrorSerializer(many=True) + meta = SourceRecordFilterOptionsMetaSerializer() + + class SourceRecordExportTicketSerializer(serializers.Serializer): ticket = serializers.CharField() expires_in = serializers.IntegerField(min_value=1) @@ -567,6 +589,11 @@ class SourceRecordExportUnavailableSerializer(serializers.Serializer): ) +class SourceRecordExportInvalidTicketSerializer(serializers.Serializer): + detail = serializers.CharField() + code = serializers.ChoiceField(choices=("invalid_ticket",)) + + class SourceRecordExportTicketExpiredSerializer(serializers.Serializer): detail = serializers.CharField() code = serializers.ChoiceField(choices=("ticket_expired",)) diff --git a/src/organizations/source_backfill.py b/src/organizations/source_backfill.py index c6d514c..96c8380 100644 --- a/src/organizations/source_backfill.py +++ b/src/organizations/source_backfill.py @@ -37,6 +37,7 @@ from organizations.source_groups import ( get_source_group_descriptor, ) from organizations.source_identity import normalize_identity_fields +from organizations.source_record_normalization import normalize_record_payload @dataclass(frozen=True) @@ -181,7 +182,13 @@ class OrganizationSourceBackfillService: "amount": adapter.amount, "status": adapter.status, "url": adapter.url, - "payload": adapter.payload, + "payload": normalize_record_payload( + adapter.source, + adapter.payload, + organization=organization, + source_group=descriptor.source_group, + persist_branch=True, + ), "load_batch": adapter.load_batch, }, ) diff --git a/src/organizations/source_ingestion.py b/src/organizations/source_ingestion.py index ddda17d..d57eba7 100644 --- a/src/organizations/source_ingestion.py +++ b/src/organizations/source_ingestion.py @@ -28,6 +28,7 @@ from organizations.source_groups import ( get_source_group_descriptor, ) from organizations.source_identity import normalize_identity_fields +from organizations.source_record_normalization import normalize_record_payload @dataclass(frozen=True) @@ -69,6 +70,7 @@ class OrganizationSourceIngestionResult: """Counters returned by direct source ingestion.""" scanned: int = 0 + staged_records: int = 0 created_organizations: int = 0 created_extensions: int = 0 updated_extensions: int = 0 @@ -107,9 +109,19 @@ class OrganizationSourceIngestionService: source: str, load_batch: int | None, records: Iterable[SourceRecordInput], + publication_run_id: str | None = None, ) -> OrganizationSourceIngestionResult: descriptor = get_source_group_descriptor(str(source)) deduplicated = cls._deduplicate_records(records) + if publication_run_id: + from apps.parsers.aggregate_publication import stage_records + + staged = stage_records( + publication_run_id, str(source), load_batch, deduplicated + ) + return OrganizationSourceIngestionResult( + scanned=staged, staged_records=staged + ) return cls._save_records(descriptor, load_batch, deduplicated) @staticmethod @@ -868,7 +880,13 @@ class OrganizationSourceIngestionService: "amount": record_input.amount, "status": str(record_input.status or ""), "url": str(record_input.url or ""), - "payload": dict(record_input.payload or {}), + "payload": normalize_record_payload( + str(descriptor.source), + record_input.payload, + organization=extension.organization, + source_group=descriptor.source_group, + persist_branch=True, + ), "load_batch": load_batch, "legacy_model": "", "legacy_pk": "", @@ -1059,7 +1077,13 @@ class OrganizationSourceIngestionService: "amount": record_input.amount, "status": str(record_input.status or ""), "url": str(record_input.url or ""), - "payload": dict(record_input.payload or {}), + "payload": normalize_record_payload( + str(descriptor.source), + record_input.payload, + organization=extension.organization, + source_group=descriptor.source_group, + persist_branch=True, + ), "load_batch": load_batch, "legacy_model": "", "legacy_pk": "", diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index e607ef9..4494711 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -190,6 +190,10 @@ class SourceRecordExportTicketInvalid(Exception): """Raised when a native-download ticket is invalid, expired, or consumed.""" +class SourceRecordExportTicketUnknown(SourceRecordExportTicketInvalid): + """Malformed ticket or a capability that was never issued.""" + + @dataclass(frozen=True) class SourceRecordExportArtifact: """One prepared source-group file in a published generation.""" @@ -566,6 +570,7 @@ def create_source_record_export_download_ticket( *, source_groups: Sequence[str], export_format: str, + user_id: int, ) -> SourceRecordExportDownloadTicket: """Validate prepared files and cache a short-lived download capability.""" @@ -578,6 +583,7 @@ def create_source_record_export_download_ticket( ) expires_in = _download_ticket_ttl() payload = { + "user_id": user_id, "sources": list(source_groups), "format": export_format, "requested_at": requested_at.isoformat(), @@ -591,6 +597,11 @@ def create_source_record_export_download_ticket( payload, timeout=expires_in, ): + cache.set( + f"{_source_record_export_ticket_cache_key(ticket)}:issued", + True, + timeout=expires_in + 86400, + ) return SourceRecordExportDownloadTicket( ticket=ticket, archive_name=package.archive_name, @@ -599,31 +610,31 @@ def create_source_record_export_download_ticket( raise RuntimeError("Could not allocate a source-record export download ticket.") +def _ticket_remaining_seconds(payload) -> float: + try: + remaining = float(payload["expires_at"]) - time.time() + except (KeyError, TypeError, ValueError): + raise SourceRecordExportTicketUnknown from None + if remaining <= 0: + raise SourceRecordExportTicketInvalid + return remaining + + def consume_source_record_export_download_ticket( ticket: str, ) -> SourceRecordExportArchive: """Consume a download ticket before streaming the prepared archive.""" if not SOURCE_RECORD_EXPORT_TICKET_PATTERN.fullmatch(ticket): - raise SourceRecordExportTicketInvalid + raise SourceRecordExportTicketUnknown cache_key = _source_record_export_ticket_cache_key(ticket) payload = cache.get(cache_key) if payload is None: - raise SourceRecordExportTicketInvalid - # Redis and LocMem add are atomic; get/delete alone lets two requests win. - try: - remaining = ( - float(payload.get("expires_at", time.time() + _download_ticket_ttl())) - - time.time() - ) - except (TypeError, ValueError, AttributeError): - raise SourceRecordExportTicketInvalid from None - if remaining <= 0 or not cache.add( - f"{cache_key}:claimed", True, timeout=max(1, math.ceil(remaining)) - ): - raise SourceRecordExportTicketInvalid - cache.delete(cache_key) + if cache.get(f"{cache_key}:issued") or cache.get(f"{cache_key}:claimed"): + raise SourceRecordExportTicketInvalid + raise SourceRecordExportTicketUnknown + _ticket_remaining_seconds(payload) try: source_groups = payload["sources"] @@ -656,12 +667,20 @@ def consume_source_record_export_download_ticket( except (KeyError, TypeError, ValueError): raise SourceRecordExportTicketInvalid from None - return build_source_records_export_archive( + package = build_source_records_export_archive( source_groups=source_groups, export_format=export_format, requested_at=requested_at, generation_id=generation_id, ) + remaining = _ticket_remaining_seconds(payload) + # Claim only after readiness validation. Atomic add prevents duplicate transfers. + if not cache.add( + f"{cache_key}:claimed", True, timeout=max(1, math.ceil(remaining)) + ): + raise SourceRecordExportTicketInvalid + cache.delete(cache_key) + return package def _source_record_export_ticket_cache_key(ticket: str) -> str: @@ -730,6 +749,10 @@ def _build_source_records_archive_name( source_name = "__".join( SOURCE_GROUP_EXPORT_FILE_STEMS[source_group] for source_group in source_groups ) + if set(source_groups) == set(SOURCE_GROUP_EXPORT_FILE_STEMS): + source_name = "organization_source_records_all" + elif len(source_name) > 150: + source_name = "organization_source_records_selected" timestamp = timezone.localtime(requested_at).strftime("%Y%m%d_%H%M%S") return f"{source_name}_{timestamp}.zip" @@ -1287,6 +1310,7 @@ def _build_record_row( "ОГРН": organization.ogrn, "КПП": organization.kpp, "ОКПО": organization.okpo, + "organization.is_branch": organization.is_branch, "uid": str(record.uid), "source_group": record.extension.source_group, "source": record.source, diff --git a/src/organizations/source_record_normalization.py b/src/organizations/source_record_normalization.py new file mode 100644 index 0000000..6b15015 --- /dev/null +++ b/src/organizations/source_record_normalization.py @@ -0,0 +1,94 @@ +"""Normalization shared by new ingestion and the published-data backfill.""" + +import logging +from datetime import date, datetime + +logger = logging.getLogger(__name__) +BRANCH_GROUPS = { + "government_procurements", + "industrial_production", + "planned_inspections", + "defense_suppliers", + "arbitration", + "security_registries", + "vacancies", +} +TERM_KEYS = ("Срок действия сертификата", "Срок действия лицензии", "Срок действия") + + +def branch_value(value): + if isinstance(value, bool): + return value + text = str(value).strip().lower() + if text in {".t.", "true", "1"}: + return True + if text in {".f.", "false", "0"}: + return False + return None + + +def canonical_branch(organization, payload, *, source): + canonical = organization.is_branch + if canonical is None: + canonical = branch_value(organization.filial) + supplied = [ + branch_value(payload[key]) for key in ("is_branch", "filial") if key in payload + ] + if canonical is None: + known = {value for value in supplied if value is not None} + if len(known) == 1: + canonical = known.pop() + if any(value is not None and value != canonical for value in supplied): + logger.warning( + "branch_conflict source=%s organization=%s; canonical organization takes precedence", + source, + organization.pk, + ) + return canonical + + +def _date_value(raw, *, source, field): + if raw is None or raw == "": + return None + if isinstance(raw, datetime): + return raw.date().isoformat() + if isinstance(raw, date): + return raw.isoformat() + text = str(raw).strip() + for format_ in ("%Y-%m-%d", "%d.%m.%Y", "%d/%m/%Y"): + try: + return datetime.strptime(text, format_).date().isoformat() + except ValueError: + pass + logger.warning("invalid_date source=%s field=%s; raw retained", source, field) + return None + + +def normalize_record_payload( + source, payload, *, organization=None, source_group=None, persist_branch=False +): + result = dict(payload or {}) + if source == "industrial": + for field in ("issue_date", "expiry_date"): + normalized_key = f"{field}_normalized" + # Imported raw is authoritative; keep already normalized legacy-only values. + raw = result[field] if field in result else result.get(normalized_key) + result[normalized_key] = _date_value(raw, source=source, field=field) + elif source == "fstec": + raw = next( + (result[key] for key in TERM_KEYS if result.get(key) not in (None, "")), + result.get("term_raw"), + ) + result["term_raw"] = raw + result["term_date"] = _date_value(raw, source=source, field="term_date") + if organization is not None and source_group in BRANCH_GROUPS: + canonical = canonical_branch(organization, result, source=source) + if persist_branch and organization.is_branch is None and canonical is not None: + type(organization).objects.filter( + pk=organization.pk, is_branch__isnull=True + ).update(is_branch=canonical) + organization.refresh_from_db(fields=["is_branch"]) + canonical = organization.is_branch + if "is_branch" in result or "filial" in result: + result["is_branch"] = canonical + return result diff --git a/src/organizations/source_record_query.py b/src/organizations/source_record_query.py index a94983e..f507ef5 100644 --- a/src/organizations/source_record_query.py +++ b/src/organizations/source_record_query.py @@ -52,6 +52,11 @@ REGISTRY_SOURCE_GROUPS = { } REGISTRY_ORDERING_FIELDS = ( + "organization__is_branch", + "payload__issue_date_normalized", + "payload__expiry_date_normalized", + "payload__term_date", + "payload__valid_from", "payload__exchange", "payload__instrument_id", "payload__instrument_kind", @@ -125,6 +130,9 @@ FEDRESURS_SECTION_STATES = ( "schema_unverified", ) DATE_RANGES = { + "first_event_date": "payload__structure__first_event_date", + "license_valid_from": "payload__valid_from", + "traded_on": "payload__trading__traded_on", "publication_date": "payload__publication_date", "license_issued": "payload__issued_on", "license_valid_until": "payload__valid_until", @@ -240,6 +248,7 @@ def query_errors(params, *, source_groups, ordering_values): _validate_choices(values, error) _validate_dates(values, error) _validate_amounts(values, error) + _validate_ordering_scope(values, error) ordering = values.get("ordering") if ordering is not None and ( not ordering or any(term not in ordering_values for term in ordering.split(",")) @@ -317,7 +326,47 @@ def _validate_choices(values, error): error("section_state") +def _validate_ordering_scope(values, error): + group = values.get("source_group") or REGISTRY_SOURCE_GROUPS.get( + values.get("source") + ) + allowed_groups = { + "payload__issue_date_normalized": "industrial_production", + "payload__expiry_date_normalized": "industrial_production", + "payload__term_date": "security_registries", + "payload__valid_from": "company_licenses", + } + if group is None: + return + for term in values.get("ordering", "").split(","): + expected = allowed_groups.get(term.lstrip("-")) + if expected is not None and group != expected: + error( + "ordering", + "invalid_ordering", + "Сортировка несовместима с выбранным источником", + ) + + +def _validate_date_scope(values, error): + group = values.get("source_group") or REGISTRY_SOURCE_GROUPS.get( + values.get("source") + ) + for prefix, allowed_group in ( + ("first_event_date", "integrated_structure_documents"), + ("license_valid_from", "company_licenses"), + ("traded_on", "stock_exchange_activity"), + ): + for suffix in ("from", "to"): + name = f"{prefix}_{suffix}" + if name in values and group != allowed_group: + error( + name, "invalid_filter", "Фильтр несовместим с выбранным источником" + ) + + def _validate_dates(values, error): + _validate_date_scope(values, error) for prefix in DATE_RANGES: valid = {} for suffix in ("from", "to"): @@ -484,6 +533,8 @@ def source_specific_search(term): def ordering_expression(field): + if field == "organization__is_branch": + return F("extension__organization__is_branch") if field in ("events_count", "documents_count"): key = "events_count" if field == "events_count" else "document_links_count" return Cast( diff --git a/src/organizations/source_record_schema.py b/src/organizations/source_record_schema.py index c51a411..355be3a 100644 --- a/src/organizations/source_record_schema.py +++ b/src/organizations/source_record_schema.py @@ -26,3 +26,11 @@ class SourceRecordContractAutoSchema(SwaggerAutoSchema): ): self.serializer_to_schema(serializer()) return super().get_operation(operation_keys) + + +class SourceRecordDownloadAutoSchema(SwaggerAutoSchema): + def get_consumes(self): + return ["application/x-www-form-urlencoded"] + + def get_produces(self): + return ["application/zip"] diff --git a/src/organizations/views.py b/src/organizations/views.py index 4fb6b06..96cac61 100644 --- a/src/organizations/views.py +++ b/src/organizations/views.py @@ -12,9 +12,9 @@ from uuid import UUID from apps.core.openapi import swagger_tag from django.conf import settings from django.core.cache import cache -from django.db.models import Case, CharField, F, Q, Value, When +from django.db.models import Case, CharField, Count, F, Max, Q, Value, When from django.db.models.fields.json import KeyTextTransform -from django.db.models.functions import Coalesce, NullIf +from django.db.models.functions import Cast, Coalesce, NullIf from django.http import FileResponse, Http404, StreamingHttpResponse from django_filters import rest_framework as filters from drf_yasg import openapi @@ -23,7 +23,7 @@ from rest_framework import status from rest_framework.decorators import action from rest_framework.exceptions import NotFound, ValidationError from rest_framework.filters import OrderingFilter, SearchFilter -from rest_framework.parsers import MultiPartParser +from rest_framework.parsers import FormParser, JSONParser, MultiPartParser from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.viewsets import ReadOnlyModelViewSet @@ -56,14 +56,17 @@ from organizations.serializers import ( OrganizationSourceRecordListSerializer, OrganizationSourceRecordSerializer, SourceRecordErrorResponseSerializer, + SourceRecordExportInvalidTicketSerializer, SourceRecordExportTicketExpiredSerializer, SourceRecordExportTicketSerializer, SourceRecordExportUnavailableSerializer, + SourceRecordFilterOptionsResponseSerializer, ) from organizations.source_record_export import ( SourceRecordExportArchive, SourceRecordExportArtifactsUnavailable, SourceRecordExportTicketInvalid, + SourceRecordExportTicketUnknown, build_source_records_export_archive, consume_source_record_export_download_ticket, create_source_record_export_download_ticket, @@ -78,7 +81,10 @@ from organizations.source_record_query import ( registry_query_parameters, source_specific_search, ) -from organizations.source_record_schema import SourceRecordContractAutoSchema +from organizations.source_record_schema import ( + SourceRecordContractAutoSchema, + SourceRecordDownloadAutoSchema, +) ORGANIZATIONS_TAG = swagger_tag("Организации", "Organizations") ORGANIZATION_SEARCH_FIELDS = ( @@ -101,12 +107,13 @@ def _query_parameter( default: str | int | bool | None = None, enum: list[str] | None = None, format_: str | None = None, + required: bool = False, ) -> openapi.Parameter: return openapi.Parameter( name=name, in_=openapi.IN_QUERY, type=param_type, - required=False, + required=required, description=description, default=default, enum=enum, @@ -611,6 +618,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): response[ "Content-Disposition" ] = f'attachment; filename="{package.archive_name}"' + response["Cache-Control"] = "no-store" response["X-Source-Export-Files"] = str(package.files_count) response["X-Source-Export-Generated-At"] = package.generated_at return response @@ -712,6 +720,101 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): self._source_record_ordering = ordering return errors + @swagger_auto_schema( + tags=[ORGANIZATIONS_TAG], + operation_id="v2_organization_source_records_filter_options", + manual_parameters=[ + _query_parameter( + "source_group", + description="Группа опубликованных записей", + enum=["leasing", "company_messages"], + required=True, + ), + _query_parameter( + "field", + description="Поле фильтра", + enum=["message_type"], + required=True, + ), + _query_parameter( + "search", description="Поиск по коду или названию, до 200 символов" + ), + ], + responses={ + 200: SourceRecordFilterOptionsResponseSerializer, + 400: SourceRecordErrorResponseSerializer, + }, + ) + @action(detail=False, methods=["get"], url_path="filter-options") + def filter_options(self, request): + params = request.query_params + if ( + params.get("source_group") not in ("leasing", "company_messages") + or params.get("field") != "message_type" + or len(params.get("search", "")) > 200 + or any( + name not in {"source_group", "field", "search"} + or len(params.getlist(name)) != 1 + for name in params + ) + ): + return Response( + { + "success": False, + "data": None, + "errors": [ + { + "code": "invalid_filter_options_request", + "message": "Некорректные параметры справочника", + } + ], + "meta": {}, + }, + status=400, + ) + # Aggregate the same visible table as list, without reading raw artifacts. + rows = ( + self.queryset.filter(extension__source_group=params["source_group"]) + .order_by() + .annotate( + value=Cast(KeyTextTransform("message_type", "payload"), CharField()) + ) + .exclude(value__isnull=True) + .exclude(value="") + .exclude(value="null") + .values("value") + .annotate( + records_count=Count("uid"), + label=Coalesce( + Max( + NullIf( + NullIf( + Cast( + KeyTextTransform("message_type_name", "payload"), + CharField(), + ), + Value(""), + ), + Value("null"), + ) + ), + F("value"), + ), + ) + ) + search = params.get("search") + if search: + rows = rows.filter(Q(value__icontains=search) | Q(label__icontains=search)) + data = list(rows.order_by("label", "value")) + return Response( + { + "success": True, + "data": data, + "errors": [], + "meta": {"total_count": len(data)}, + } + ) + @classmethod def _filter_search_queryset(cls, queryset, search_terms: list[str]): for search_term in search_terms: @@ -920,6 +1023,7 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): download_ticket = create_source_record_export_download_ticket( source_groups=serializer.validated_data["sources"], export_format=serializer.validated_data["format"], + user_id=request.user.pk, ) except SourceRecordExportArtifactsUnavailable: return Response( @@ -953,18 +1057,28 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): tags=[ORGANIZATIONS_TAG], operation_id="v2_organization_source_records_export_download", operation_summary="Скачать готовые записи источников по ticket", - request_body=OrganizationSourceRecordExportDownloadSerializer, + manual_parameters=[ + openapi.Parameter( + "ticket", openapi.IN_FORM, type=openapi.TYPE_STRING, required=True + ) + ], + auto_schema=SourceRecordDownloadAutoSchema, responses={ 200: openapi.Response( description="Потоковый ZIP-архив записей источников.", schema=openapi.Schema(type=openapi.TYPE_FILE), ), - 400: "Ticket отсутствует или имеет неверный формат.", + 400: SourceRecordExportInvalidTicketSerializer, 410: SourceRecordExportTicketExpiredSerializer, 503: SourceRecordExportUnavailableSerializer, }, ) - @action(detail=False, methods=["post"], url_path="export-download") + @action( + detail=False, + methods=["post"], + url_path="export-download", + parser_classes=[FormParser, MultiPartParser, JSONParser], + ) def export_download( self, request, @@ -972,12 +1086,21 @@ class OrganizationSourceRecordViewSet(ReadOnlyModelViewSet): **kwargs: Any, ) -> StreamingHttpResponse | Response: serializer = OrganizationSourceRecordExportDownloadSerializer(data=request.data) - serializer.is_valid(raise_exception=True) + if not serializer.is_valid(): + return Response( + {"detail": "Некорректный ticket скачивания.", "code": "invalid_ticket"}, + status=400, + ) try: package = consume_source_record_export_download_ticket( serializer.validated_data["ticket"] ) + except SourceRecordExportTicketUnknown: + return Response( + {"detail": "Некорректный ticket скачивания.", "code": "invalid_ticket"}, + status=400, + ) except SourceRecordExportTicketInvalid: return Response( { diff --git a/tests/apps/core/test_admin.py b/tests/apps/core/test_admin.py index c38e739..a1b4ac3 100644 --- a/tests/apps/core/test_admin.py +++ b/tests/apps/core/test_admin.py @@ -102,7 +102,7 @@ class CoreAdminTest(TestCase): job = BackgroundJob.objects.create( task_id=fake.uuid4(), task_name="test.task", - status="pending", + status="queued", ) request = self._request() qs = BackgroundJob.objects.all() @@ -110,7 +110,7 @@ class CoreAdminTest(TestCase): self.admin.revoke_jobs(request, qs) revoke_mock.assert_called_once_with(job.task_id, terminate=True) job.refresh_from_db() - self.assertEqual(job.status, "revoked") + self.assertEqual(job.status, "cancelled") class AdminDashboardTest(TestCase): diff --git a/tests/apps/core/test_job_acceptance_lifecycle.py b/tests/apps/core/test_job_acceptance_lifecycle.py new file mode 100644 index 0000000..fa28b18 --- /dev/null +++ b/tests/apps/core/test_job_acceptance_lifecycle.py @@ -0,0 +1,107 @@ +"""Canonical terminal states are shared by jobs and source cards.""" + +import pytest +from apps.core.models import BackgroundJob +from apps.core.serializers import BackgroundJobSerializer +from apps.parsers.source_cards import SOURCE_CARD_BY_SLUG, SourceCardService +from django.core.cache import cache + + +@pytest.mark.django_db +def test_new_job_uses_canonical_states_and_terminal_failure_is_private(): + cache.clear() + task_name = SOURCE_CARD_BY_SLUG["planned-inspections"].task_names[0] + job = BackgroundJob.objects.create(task_id="acceptance-failed", task_name=task_name) + assert job.status == "queued" + job.mark_started() + assert job.status == "running" + job.update_progress(100, "Publishing") + assert job.progress == 99 + job.fail("HTTPError: https://user:secret@example.test/private?token=secret") + for _ in range(2): + payload = BackgroundJobSerializer(job).data + assert payload["status"] == "failed" + assert "secret" not in str(payload) + assert "HTTPError" not in str(payload) + card = SourceCardService.get_card("planned-inspections") + assert card["status"] == "failed" + assert card["active_tasks"] == [] + assert "HTTPError" not in card["error_message"] + assert "secret" not in card["error_message"] + job.complete() + assert job.status == "failed" + + +@pytest.mark.django_db +def test_success_and_skipped_jobs_are_terminal_and_card_progress_is_100(): + cache.clear() + name = SOURCE_CARD_BY_SLUG["planned-inspections"].task_names[0] + job = BackgroundJob.objects.create( + task_id="acceptance-success", task_name=name, user_id=1 + ) + job.complete(result={"status": "skipped"}) + assert job.status == "skipped" + assert job.is_finished and job.is_successful + from apps.core.services import BackgroundJobService + + assert job.pk in [ + item.pk + for item in BackgroundJobService.get_user_jobs(job.user_id, status="skipped") + ] + assert BackgroundJobSerializer(job).data["status"] == "skipped" + card = SourceCardService.get_card("planned-inspections") + assert card["status"] == "skipped" + assert card["progress"] == 100 + assert card["active_tasks"] == [] + + +@pytest.mark.django_db +def test_status_data_migration_maps_legacy_rows_and_supports_rollback(): + from importlib import import_module + from types import SimpleNamespace + + from django.apps import apps + from django.db import connection + + migration = import_module("apps.core.migrations.0002_canonical_job_statuses") + for index, status in enumerate(migration.STATUS_MAP): + BackgroundJob.objects.create( + task_id=f"migration-{index}", task_name="test", status=status + ) + BackgroundJob.objects.create( + task_id="migration-skipped", + task_name="test", + status="success", + result={"status": "skipped"}, + ) + editor = SimpleNamespace(connection=connection) + migration.forward_statuses(apps, editor) + assert set(BackgroundJob.objects.values_list("status", flat=True)) == set( + migration.STATUS_MAP.values() + ) | {"skipped"} + migration.reverse_statuses(apps, editor) + assert set(BackgroundJob.objects.values_list("status", flat=True)) == { + "pending", + "started", + "retry", + "failure", + "revoked", + "success", + } + + +@pytest.mark.django_db +def test_skipped_load_does_not_advance_last_successful_publication(): + from apps.parsers.models import ParserLoadLog + + cache.clear() + success = ParserLoadLog.objects.create( + source="inspections", batch_id=1, status="success", records_count=10 + ) + ParserLoadLog.objects.create( + source="inspections", batch_id=2, status="skipped", records_count=0 + ) + card = SourceCardService.get_card("planned-inspections") + assert card["status"] == "skipped" + assert card["last_updated_at"] == success.updated_at + assert card["latest_success_load"]["status"] == "success" diff --git a/tests/apps/core/test_job_control_race.py b/tests/apps/core/test_job_control_race.py index 7730b78..f1b6811 100644 --- a/tests/apps/core/test_job_control_race.py +++ b/tests/apps/core/test_job_control_race.py @@ -46,9 +46,13 @@ def test_revoke_preserves_worker_terminal_result(terminal): assert response.status_code == 200 job.refresh_from_db() assert {field: getattr(job, field) for field in expected} == expected - assert response.data["status"] == ("success" if terminal == "success" else "error") + assert response.data["status"] == ("success" if terminal == "success" else "failed") assert response.data["progress"] == expected["progress"] - assert response.data["message"] == expected["progress_message"] + assert response.data["message"] == ( + expected["progress_message"] + if terminal == "success" + else "Не удалось обновить данные. Повторите попытку позже." + ) @pytest.mark.django_db diff --git a/tests/apps/organizations/test_canonical_openapi.py b/tests/apps/organizations/test_canonical_openapi.py index d955024..00d6c9f 100644 --- a/tests/apps/organizations/test_canonical_openapi.py +++ b/tests/apps/organizations/test_canonical_openapi.py @@ -193,3 +193,22 @@ def test_fssp_variant_preserves_signed_nullable_amounts(contract): "ended", "terminated", ] + + +def test_testing_amendments_download_and_options_contract(contract): + download = contract["paths"][ + "/api/v2/organization-source-records/export-download/" + ]["post"] + assert set(download["requestBody"]["content"]) == { + "application/x-www-form-urlencoded" + } + assert set(download["responses"]["200"]["content"]) == {"application/zip"} + invalid = download["responses"]["400"]["content"]["application/json"]["schema"] + schema = contract["components"]["schemas"][invalid["$ref"].rsplit("/", 1)[1]] + assert schema["properties"]["code"]["enum"] == ["invalid_ticket"] + options = contract["paths"]["/api/v2/organization-source-records/filter-options/"][ + "get" + ] + assert { + param["name"] for param in options["parameters"] if param.get("required") + } == {"source_group", "field"} diff --git a/tests/apps/organizations/test_checko_section_contract.py b/tests/apps/organizations/test_checko_section_contract.py index 97da5e6..6057a69 100644 --- a/tests/apps/organizations/test_checko_section_contract.py +++ b/tests/apps/organizations/test_checko_section_contract.py @@ -178,7 +178,7 @@ class CheckoSectionContractTests(TestCase): with pytest.raises(SnapshotValidationError, match="no_completed_organizations"): self.run_task(source) job = BackgroundJob.objects.get() - assert job.status == "failure" + assert job.status == "failed" assert job.result["quota_skipped"] == 1 assert job.result["organizations_preserved"] == 1 assert job.result["completeness"] == "partial" diff --git a/tests/apps/organizations/test_openapi_runtime_fixtures.py b/tests/apps/organizations/test_openapi_runtime_fixtures.py index 21b918b..f1972f0 100644 --- a/tests/apps/organizations/test_openapi_runtime_fixtures.py +++ b/tests/apps/organizations/test_openapi_runtime_fixtures.py @@ -9,6 +9,7 @@ from unittest.mock import patch from uuid import uuid4 import pytest +from apps.core.models import BackgroundJob from apps.parsers.budget_registry import normalize_budget_record from apps.parsers.checko_sections import collect_checko_section from apps.parsers.fssp_csv import FSSP_SOURCE, GENERAL_DATASET, iter_fssp_records @@ -333,6 +334,31 @@ def test_source_record_runtime_fixtures(): 404, ) + options_url = list_url + "filter-options/" + capture( + "message-type-options", + "options", + client.get(options_url, {"source_group": "leasing", "field": "message_type"}), + 200, + ) + capture( + "message-type-options-invalid", + "options", + client.get(options_url, {"source_group": "unknown", "field": "message_type"}), + 400, + ) + user = UserFactory.create_superuser() + client.force_authenticate(user) + for state in ("failed", "success", "skipped"): + job = BackgroundJob.objects.create( + task_id=f"contract-{state}", task_name="test", user_id=user.id + ) + if state == "failed": + job.fail("HTTPError: https://user:secret@example.test/?token=secret") + else: + job.complete(result={"status": state}) + capture(f"job-{state}", "job", client.get(f"/api/v1/jobs/{job.task_id}/"), 200) + # Ordinary pytest runs create no artifact. CI opts in with an explicit path. destination = os.environ.get("OPENAPI_RUNTIME_FIXTURES") if destination: diff --git a/tests/apps/organizations/test_source_export_snapshot_contract.py b/tests/apps/organizations/test_source_export_snapshot_contract.py index 5d06c1e..ebf8741 100644 --- a/tests/apps/organizations/test_source_export_snapshot_contract.py +++ b/tests/apps/organizations/test_source_export_snapshot_contract.py @@ -158,7 +158,7 @@ def test_ticket_expiry_and_reuse_are_typed_410(export_client): assert repeated.status_code == 410 assert repeated.data["code"] == "ticket_expired" pending = exports.create_source_record_export_download_ticket( - source_groups=["budget_process_registry"], export_format="json" + source_groups=["budget_process_registry"], export_format="json", user_id=1 ) with patch.object(exports.time, "time", return_value=time.time() + 301): expired = export_client.post(download_url(), {"ticket": pending.ticket}) diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index 1cce620..fd7261c 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -723,11 +723,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): with self.assertNumQueries(0): download_response = self.client.post( self.download_url, - {"ticket": ticket_response.data["ticket"]}, - format="multipart", + "ticket=" + ticket_response.data["ticket"], + content_type="application/x-www-form-urlencoded", ) self.assertEqual(download_response.status_code, status.HTTP_200_OK) + self.assertEqual(download_response["Cache-Control"], "no-store") self.assertTrue(download_response.streaming) self.assertEqual(download_response["Content-Type"], "application/zip") self.assertNotIn("Content-Length", download_response) @@ -751,6 +752,72 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): "ticket_expired", ) + def test_all_sources_filename_is_short_and_invalid_ticket_is_400(self): + from organizations.source_record_export import ( + _build_source_records_archive_name, + ) + + name = _build_source_records_archive_name( + source_groups=SourceGroup.values, + requested_at=datetime(2026, 10, 9, tzinfo=UTC), + ) + self.assertRegex(name, r"^organization_source_records_all_.*\.zip$") + self.assertLess(len(name), 100) + self.client.force_authenticate(user=None) + response = self.client.post( + self.download_url, + "ticket=" + "x" * 43, + content_type="application/x-www-form-urlencoded", + ) + self.assertEqual(response.status_code, 400) + self.assertEqual(response.data["code"], "invalid_ticket") + + def test_download_not_ready_does_not_consume_ticket(self): + from organizations.source_record_export import ( + SourceRecordExportArtifactsUnavailable, + ) + + self.client.force_authenticate(UserFactory.create_superuser()) + build_source_record_export_artifacts() + response = self.client.post( + self.ticket_url, + {"sources": ["planned_inspections"], "format": "json"}, + format="json", + ) + self.assertEqual(response.status_code, 201) + ticket = response.data["ticket"] + self.client.force_authenticate(user=None) + with patch( + "organizations.source_record_export.build_source_records_export_archive", + side_effect=SourceRecordExportArtifactsUnavailable, + ): + unavailable = self.client.post(self.download_url, {"ticket": ticket}) + self.assertEqual(unavailable.status_code, 503) + ready = self.client.post(self.download_url, {"ticket": ticket}) + self.assertEqual(ready.status_code, 200) + self._response_body(ready) + + def test_all_sources_native_form_download_can_be_repeated_with_new_ticket(self): + build_source_record_export_artifacts() + for _ in range(2): + self.client.force_authenticate(UserFactory.create_superuser()) + response = self.client.post( + self.ticket_url, + {"sources": SourceGroup.values, "format": "json"}, + format="json", + ) + self.assertEqual(response.status_code, 201) + self.assertLess(len(response.data["file_name"]), 100) + self.client.force_authenticate(user=None) + download = self.client.post( + self.download_url, + "ticket=" + response.data["ticket"], + content_type="application/x-www-form-urlencoded", + ) + self.assertEqual(download.status_code, 200) + with zipfile.ZipFile(BytesIO(self._response_body(download))) as archive: + self.assertEqual(len(archive.namelist()), len(SourceGroup.values)) + def test_csv_export_uses_bom_and_canonical_columns_before_payload(self): self.client.force_authenticate(UserFactory.create_superuser()) organization = Organization.objects.create( diff --git a/tests/apps/organizations/test_testing_amendments.py b/tests/apps/organizations/test_testing_amendments.py new file mode 100644 index 0000000..36fc7a0 --- /dev/null +++ b/tests/apps/organizations/test_testing_amendments.py @@ -0,0 +1,387 @@ +"""Regression cases from the 8 October backend acceptance report.""" + +import json +from io import StringIO +from uuid import UUID + +import pytest +from django.core.management import call_command +from django.utils import timezone +from organizations.models import ( + Organization, + OrganizationSourceExtension, + OrganizationSourceRecord, +) +from organizations.source_ingestion import ( + OrganizationSourceIngestionService, + SourceRecordInput, +) +from organizations.source_record_export import _build_record_row +from rest_framework.test import APIClient + +from tests.apps.user.factories import UserFactory + +LIST_URL = "/api/v2/organization-source-records/" + + +@pytest.fixture +def client(db): + client = APIClient() + client.force_authenticate(UserFactory.create_user()) + return client + + +def record(group, source, index, payload=None, **organization_fields): + supplied = payload or {} + if source == "checko_licenses": + payload = { + "license_number": "123", + "issued_on": "2026-01-01", + "valid_from": None, + "valid_until": None, + "authority": "Authority", + "activities": [], + "source_url": "https://example.test", + "fetched_at": "2026-10-09T00:00:00Z", + "upstream": {}, + **supplied, + } + if payload["valid_from"] == "": + payload["valid_from"] = None + elif source in ("checko_leasing", "checko_messages"): + payload = { + "message_guid": str(index), + "publication_raw": None, + "publication_date": None, + "message_type": "", + "message_type_name": None, + "publisher_identifier": None, + "participant_identifiers": [], + "source_url": None, + "fetched_at": "2026-10-09T00:00:00Z", + "upstream": {}, + "lease_event_type": "test", + **supplied, + } + elif source == "stock_exchange_activity": + payload = { + "exchange": "moex", + "instrument_id": str(index), + "issuer_inn": "7707083810", + "issuer_provider_id": None, + "isin": None, + "instrument_kind": "stock", + "listing_date": None, + "nominal_value": None, + "nominal_currency": None, + "parameters": {}, + "duration": None, + "yield": None, + "provenance": {}, + "upstream": {}, + **supplied, + } + payload["trading"] = {"board_id": "test", "upstream": {}, **payload["trading"]} + organization = Organization.objects.create( + name=f"Organization {index}", + opk_registry_membership=True, + **organization_fields, + ) + extension = OrganizationSourceExtension.objects.create( + organization=organization, source_group=group + ) + return OrganizationSourceRecord.objects.create( + uid=UUID(int=index), + extension=extension, + source=source, + record_type="test", + external_id=str(index), + payload=payload or {}, + record_date="2026-01-01", + ) + + +@pytest.mark.parametrize( + "ordering,expected", + [ + ("organization__is_branch", [3, 1, 2]), + ("-organization__is_branch", [1, 3, 2]), + ], +) +@pytest.mark.parametrize( + "group,source", + [ + ("government_procurements", "procurements_44fz"), + ("industrial_production", "industrial"), + ("planned_inspections", "inspections"), + ("defense_suppliers", "fas_goz"), + ("arbitration", "arbitration"), + ("security_registries", "fstec"), + ("vacancies", "trudvsem"), + ], +) +def test_branch_ordering_is_boolean_null_last_before_pagination( + client, ordering, expected, group, source +): + record(group, source, 1, is_branch=True) + record(group, source, 2, is_branch=None) + record(group, source, 3, is_branch=False) + for page, expected_id in enumerate(expected, 1): + response = client.get( + LIST_URL, {"ordering": ordering, "page_size": 1, "page": page} + ) + assert response.status_code == 200, response.data + assert response.data["data"][0]["external_id"] == str(expected_id) + + +@pytest.mark.parametrize( + "group,source,prefix,payloads", + [ + ( + "integrated_structure_documents", + "integrated_structures", + "first_event_date", + [ + {"structure": {"first_event_date": date}} + for date in ["2026-10-08", "2026-10-09", None] + ], + ), + ( + "company_licenses", + "checko_licenses", + "license_valid_from", + [{"valid_from": date} for date in ["2026-10-08", "2026-10-09", None]], + ), + ( + "stock_exchange_activity", + "stock_exchange_activity", + "traded_on", + [ + {"exchange": exchange, "trading": {"traded_on": date}} + for exchange, date in [ + ("moex", "2026-10-08"), + ("spb", "2026-10-09"), + ("moex", None), + ] + ], + ), + ], +) +def test_date_ranges_use_business_date_and_include_boundaries( + client, group, source, prefix, payloads +): + for index, payload in enumerate(payloads, 1): + record(group, source, index, payload) + response = client.get( + LIST_URL, + { + "source_group": group, + f"{prefix}_from": "2026-10-08", + f"{prefix}_to": "2026-10-09", + "ordering": "uid", + }, + ) + assert response.status_code == 200, response.data + assert [item["external_id"] for item in response.data["data"]] == ["1", "2"] + for params, code in [ + ( + {f"{prefix}_from": "2026-10-10", f"{prefix}_to": "2026-10-09"}, + "invalid_date_range", + ), + ({f"{prefix}_from": "09.10.2026"}, "invalid_date"), + ( + {"source_group": "arbitration", f"{prefix}_from": "2026-10-09"}, + "invalid_filter", + ), + ]: + invalid = client.get(LIST_URL, {"source_group": group, **params}) + assert invalid.status_code == 400 + assert code in [error["code"] for error in invalid.data["errors"]] + + +def test_message_type_options_count_the_same_visible_records(client): + record( + "leasing", + "checko_leasing", + 1, + {"message_type": "Lease", "message_type_name": "Лизинг"}, + ) + record( + "leasing", + "checko_leasing", + 2, + {"message_type": "Lease", "message_type_name": "Лизинг"}, + ) + record("leasing", "checko_leasing", 3, {"message_type": "Other"}) + record("company_messages", "checko_messages", 4, {"message_type": "Lease"}) + response = client.get( + LIST_URL + "filter-options/", + {"source_group": "leasing", "field": "message_type", "search": "Лиз"}, + ) + assert response.status_code == 200, response.data + assert response.data["data"] == [ + {"value": "Lease", "label": "Лизинг", "records_count": 2} + ] + filtered = client.get( + LIST_URL, {"source_group": "leasing", "message_type": "Lease"} + ) + assert filtered.data["meta"]["pagination"]["total_count"] == 2 + assert ( + client.get( + LIST_URL + "filter-options/", + {"source_group": "arbitration", "field": "message_type"}, + ).status_code + == 400 + ) + + +@pytest.mark.parametrize( + "source,group,key", + [ + ("industrial", "industrial_production", "issue_date_normalized"), + ("industrial", "industrial_production", "expiry_date_normalized"), + ("fstec", "security_registries", "term_date"), + ("checko_licenses", "company_licenses", "valid_from"), + ], +) +def test_normalized_date_ordering_null_last(client, source, group, key): + for index, value in enumerate(["2027-01-01", None, "2026-12-31", ""], 1): + record(group, source, index, {key: value}) + invalid = client.get( + LIST_URL, {"source_group": "arbitration", "ordering": "payload__" + key} + ) + assert invalid.status_code == 400 + for ordering, expected in [(key, [3, 1, 2, 4]), ("-" + key, [1, 3, 2, 4])]: + response = client.get( + LIST_URL, + { + "source_group": group, + "source": source, + "ordering": ("-" if ordering.startswith("-") else "") + + "payload__" + + key, + }, + ) + assert response.status_code == 200, response.data + assert [int(item["external_id"]) for item in response.data["data"]] == expected + + +@pytest.mark.parametrize( + "raw_key", ["Срок действия сертификата", "Срок действия лицензии", "Срок действия"] +) +def test_fstec_ingestion_publishes_normalized_term_without_losing_raw(client, raw_key): + organization = Organization.objects.create( + name="FSTEC", + inn="7707083810", + directory_imported_at=timezone.now(), + opk_registry_membership=True, + ) + OrganizationSourceIngestionService.save_records( + source="fstec", + load_batch=1, + records=[ + SourceRecordInput( + external_id="term", + title="Term", + organization_name="FSTEC", + inn=organization.inn, + payload={raw_key: "31.12.2027"}, + ) + ], + ) + saved = OrganizationSourceRecord.objects.get() + assert saved.payload["term_date"] == "2027-12-31" + assert saved.payload["term_raw"] == "31.12.2027" + assert saved.payload[raw_key] == "31.12.2027" + for path in [LIST_URL, LIST_URL + str(saved.uid) + "/"]: + response = client.get(path) + assert response.status_code == 200 + data = response.data["data"] if path == LIST_URL else response.data + item = data[0] if isinstance(data, list) else data + assert item["payload"]["term_date"] == "2027-12-31" + assert ( + _build_record_row(saved, include_financial_lines=False)["payload.term_date"] + == "2027-12-31" + ) + + +def test_normalization_backfill_is_dry_run_idempotent_and_preserves_identity( + client, caplog +): + saved = record( + "security_registries", + "fstec", + 1, + {"Срок действия": "неизвестен", "filial": ".F."}, + filial=".T.", + is_branch=None, + ) + original_updated_at = saved.updated_at + output = StringIO() + call_command("normalize_testing_amendments", stdout=output) + assert json.loads(output.getvalue())["changed_organizations"] == 1 + saved.refresh_from_db() + assert "term_date" not in saved.payload + call_command("normalize_testing_amendments", apply=True, stdout=StringIO()) + saved.refresh_from_db() + saved.extension.organization.refresh_from_db() + assert saved.extension.organization.is_branch is True + assert saved.payload["term_date"] is None + assert saved.payload["term_raw"] == "неизвестен" + assert ( + _build_record_row(saved, include_financial_lines=False)[ + "organization.is_branch" + ] + is True + ) + assert saved.payload["is_branch"] is True + assert saved.payload["filial"] == ".F." + assert saved.updated_at == original_updated_at + assert "invalid_date" in caplog.text + assert "branch_conflict" in caplog.text + second = StringIO() + call_command("normalize_testing_amendments", apply=True, stdout=second) + assert json.loads(second.getvalue())["changed_records"] == 0 + + +def test_industrial_ingestion_normalizes_dates_and_logs_invalid_values(client, caplog): + organization = Organization.objects.create( + name="Industrial", inn="7707083810", directory_imported_at=timezone.now() + ) + OrganizationSourceIngestionService.save_records( + source="industrial", + load_batch=1, + records=[ + SourceRecordInput( + external_id="certificate", + title="Certificate", + organization_name="Industrial", + inn=organization.inn, + payload={"issue_date": "31.12.2025", "expiry_date": "wrong"}, + ) + ], + ) + saved = OrganizationSourceRecord.objects.get() + assert saved.payload["issue_date_normalized"] == "2025-12-31" + assert saved.payload["expiry_date_normalized"] is None + assert saved.payload["expiry_date"] == "wrong" + assert "invalid_date" in caplog.text + + +@pytest.mark.parametrize( + "group,source", + [("government_procurements", "procurements_44fz"), ("arbitration", "arbitration")], +) +@pytest.mark.parametrize( + "ordering,expected", [("amount", [2, 1, 3]), ("-amount", [1, 2, 3])] +) +def test_amount_ordering_remains_decimal_null_last( + client, group, source, ordering, expected +): + for index, amount in enumerate(["100.00", "20.00", None], 1): + saved = record(group, source, index) + saved.amount = amount + saved.save(update_fields=["amount"]) + response = client.get(LIST_URL, {"source_group": group, "ordering": ordering}) + assert response.status_code == 200, response.data + assert [int(item["external_id"]) for item in response.data["data"]] == expected diff --git a/tests/apps/parsers/test_aggregate_publication.py b/tests/apps/parsers/test_aggregate_publication.py new file mode 100644 index 0000000..cc3cf2b --- /dev/null +++ b/tests/apps/parsers/test_aggregate_publication.py @@ -0,0 +1,386 @@ +"""A composed refresh exposes all parts together or preserves the old records.""" +from unittest.mock import patch +from uuid import uuid4 + +import pytest +from apps.core.models import BackgroundJob +from apps.parsers.aggregate_publication import finish_part +from apps.parsers.models import ParserLoadLog, ParserRefreshRun, ParserRefreshStage +from apps.parsers.source_cards import SourceCardService +from organizations.models import Organization, OrganizationSourceRecord +from organizations.source_ingestion import ( + OrganizationSourceIngestionService, + SourceRecordInput, +) + + +@pytest.fixture +def run(db): + organization = Organization.objects.create( + name="Factory", + inn="7707083810", + directory_imported_at="2026-10-09T00:00:00Z", + opk_registry_membership=True, + ) + tasks = {"industrial": str(uuid4()), "manufactures": str(uuid4())} + run = ParserRefreshRun.objects.create( + source_card="manufacturers-and-products", tasks=tasks + ) + jobs = {} + logs = {} + for source, task_id in tasks.items(): + jobs[source] = BackgroundJob.objects.create( + task_id=task_id, + task_name=f"test.{source}", + meta={ + "source": source, + "aggregate_refresh_id": str(run.pk), + "source_card": run.source_card, + "refresh_task_ids": list(tasks.values()), + }, + ) + jobs[source].mark_started() + logs[source] = ParserLoadLog.objects.create( + source=source, batch_id=1, status="in_progress" + ) + OrganizationSourceIngestionService.save_records( + source=source, + load_batch=0, + records=[ + SourceRecordInput( + external_id="old", + title="Old", + organization_name=organization.name, + inn=organization.inn, + ) + ], + ) + return run, jobs, logs, organization + + +def stage(run, source, organization): + return OrganizationSourceIngestionService.save_records( + source=source, + load_batch=1, + publication_run_id=str(run.pk), + records=[ + SourceRecordInput( + external_id="new", + title="New", + organization_name=organization.name, + inn=organization.inn, + payload={"issue_date": "09.10.2026"}, + ) + ], + ) + + +def ready(run, source, jobs, logs): + return finish_part( + jobs[source], logs[source], {"status": "success", "batch_id": 1, "saved": 1} + ) + + +def test_all_parts_and_terminal_jobs_are_published_together(run): + run, jobs, logs, organization = run + stage(run, "industrial", organization) + ready(run, "industrial", jobs, logs) + assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() + jobs["industrial"].refresh_from_db() + assert jobs["industrial"].status == "running" + logs["industrial"].refresh_from_db() + assert logs["industrial"].status == "in_progress" + stage(run, "manufactures", organization) + ready(run, "manufactures", jobs, logs) + assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} + assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"} + run.refresh_from_db() + assert run.status == "published" + assert not ParserRefreshStage.objects.exists() + assert SourceCardService.get_card(run.source_card)["active_tasks"] == [] + + +def test_part_failure_discards_staged_data_and_closes_all_jobs(run): + run, jobs, logs, organization = run + before = list( + OrganizationSourceRecord.objects.values("uid", "title", "payload", "updated_at") + ) + stage(run, "industrial", organization) + ready(run, "industrial", jobs, logs) + jobs["manufactures"].fail("HTTPError: secret") + assert ( + list( + OrganizationSourceRecord.objects.values( + "uid", "title", "payload", "updated_at" + ) + ) + == before + ) + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"} + assert not ParserRefreshStage.objects.exists() + assert SourceCardService.get_card(run.source_card)["active_tasks"] == [] + with pytest.raises(RuntimeError): + stage(run, "manufactures", organization) + + +def test_publication_failure_rolls_back_every_part(run): + run, jobs, logs, organization = run + stage(run, "industrial", organization) + ready(run, "industrial", jobs, logs) + stage(run, "manufactures", organization) + original = OrganizationSourceIngestionService.save_records + + def save(**kwargs): + if kwargs["source"] == "manufactures": + raise RuntimeError("database publication failure") + return original(**kwargs) + + with patch.object( + OrganizationSourceIngestionService, "save_records", side_effect=save + ), pytest.raises(RuntimeError): + ready(run, "manufactures", jobs, logs) + assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"failed"} + + +def test_cancellation_closes_siblings_and_late_delivery_cannot_publish(run): + from apps.parsers.aggregate_publication import claim_part + + run, jobs, logs, organization = run + stage(run, "industrial", organization) + ready(run, "industrial", jobs, logs) + jobs["manufactures"].revoke() + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"cancelled"} + assert claim_part(jobs["manufactures"].task_id)["status"] == "cancelled" + assert not ParserRefreshStage.objects.exists() + assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() + + +def test_duplicate_delivery_and_failure_after_commit_are_noops(run): + from apps.parsers.aggregate_publication import claim_part + + run, jobs, logs, organization = run + assert claim_part(jobs["industrial"].task_id) is None + assert claim_part(jobs["industrial"].task_id)["duplicate_delivery"] is True + for source in jobs: + stage(run, source, organization) + ready(run, source, jobs, logs) + jobs["industrial"].fail("late failure") + assert jobs["industrial"].status == "success" + assert claim_part(jobs["industrial"].task_id)["status"] == "success" + assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 + + +def test_real_parser_tasks_stage_and_publish_through_the_barrier(run): + from types import SimpleNamespace + + from apps.parsers.tasks import parse_industrial_production, parse_manufactures + + run, jobs, logs, organization = run + certificate = SimpleNamespace( + certificate_number="new", + organisation_name=organization.name, + inn=organization.inn, + ogrn="", + issue_date="09.10.2026", + expiry_date="09.10.2027", + certificate_file_url="", + ) + manufacturer = SimpleNamespace( + inn=organization.inn, + ogrn="", + full_legal_name=organization.name, + address="Factory address", + ) + with patch("apps.parsers.tasks.IndustrialProductionClient") as client: + client.return_value.__enter__.return_value.fetch_certificates.return_value = [ + certificate + ] + result = parse_industrial_production.apply( + task_id=jobs["industrial"].task_id, kwargs={"proxies": []}, throw=True + ).result + assert result["publication_pending"] is True + assert not OrganizationSourceRecord.objects.filter(external_id="new").exists() + with patch("apps.parsers.tasks.ManufacturesClient") as client: + client.return_value.__enter__.return_value.fetch_manufacturers.return_value = [ + manufacturer + ] + result = parse_manufactures.apply( + task_id=jobs["manufactures"].task_id, kwargs={"proxies": []}, throw=True + ).result + assert result["status"] == "success" + assert ( + OrganizationSourceRecord.objects.filter( + source="industrial", external_id="new" + ).count() + == 1 + ) + assert ( + OrganizationSourceRecord.objects.filter( + source="manufactures", external_id=organization.inn + ).count() + == 1 + ) + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} + + +@pytest.mark.django_db(transaction=True) +def test_concurrent_last_parts_publish_once_on_postgres(run): + from concurrent.futures import ThreadPoolExecutor + + from django.db import close_old_connections, connection + + if connection.vendor != "postgresql": + pytest.skip("Requires PostgreSQL row locks and independent connections") + run, jobs, logs, organization = run + for source in jobs: + stage(run, source, organization) + + def finish(source): + close_old_connections() + try: + return finish_part( + BackgroundJob.objects.get(pk=jobs[source].pk), + ParserLoadLog.objects.get(pk=logs[source].pk), + {"status": "success", "saved": 1, "batch_id": 1}, + ) + finally: + close_old_connections() + + with ThreadPoolExecutor(max_workers=2) as workers: + results = list(workers.map(finish, jobs)) + assert [result["status"] for result in results].count("success") == 1 + assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} + + +@pytest.mark.django_db(transaction=True) +def test_reader_sees_previous_snapshot_until_commit_on_postgres(run): + from concurrent.futures import ThreadPoolExecutor + + from django.db import close_old_connections, connection + + if connection.vendor != "postgresql": + pytest.skip("Requires independent PostgreSQL reader") + run, jobs, logs, organization = run + for source in jobs: + stage(run, source, organization) + ready(run, "industrial", jobs, logs) + original = OrganizationSourceIngestionService.save_records + + def read(): + close_old_connections() + try: + return OrganizationSourceRecord.objects.filter( + external_id="new" + ).count(), set(BackgroundJob.objects.values_list("status", flat=True)) + finally: + close_old_connections() + + def save(**kwargs): + result = original(**kwargs) + with ThreadPoolExecutor(max_workers=1) as reader: + assert reader.submit(read).result(timeout=15) == (0, {"running"}) + return result + + with patch.object( + OrganizationSourceIngestionService, "save_records", side_effect=save + ): + ready(run, "manufactures", jobs, logs) + assert OrganizationSourceRecord.objects.filter(external_id="new").count() == 2 + + +@pytest.mark.parametrize("fail_second", [False, True]) +def test_defense_refresh_tasks_share_one_publication_boundary(db, fail_second): + from types import SimpleNamespace + + from apps.parsers.clients.common import GenericParserItem + from apps.parsers.tasks import _run_generic_parser + + organization = Organization.objects.create( + name="Supplier", + inn="7707083810", + directory_imported_at="2026-10-09T00:00:00Z", + opk_registry_membership=True, + ) + + def enqueue(**kwargs): + return {"task_id": kwargs["task_id"], "task_name": kwargs["task_name"]} + + with patch.object(SourceCardService, "_enqueue_task", side_effect=enqueue): + SourceCardService._launch_refresh( + SourceCardService.get_definition("defense-unreliable-suppliers"), + requested_by_id=None, + params={}, + ) + run = ParserRefreshRun.objects.get() + for index, source in enumerate(run.tasks): + + def fetch(source=source, index=index): + if fail_second and index == 1: + raise RuntimeError("upstream failed") + return [ + GenericParserItem( + source=source, + external_id="new", + inn=organization.inn, + organisation_name=organization.name, + ) + ] + + job = BackgroundJob.objects.get(task_id=run.tasks[source]) + + def execute(job=job, source=source, fetch=fetch): + return _run_generic_parser( + SimpleNamespace(request=SimpleNamespace(id=job.task_id)), + source_key=source, + source=source, + task_name=job.task_name, + fetch_records=fetch, + ) + + if fail_second and index == 1: + with pytest.raises(RuntimeError): + execute() + else: + execute() + if index == 0: + assert not OrganizationSourceRecord.objects.exists() + assert OrganizationSourceRecord.objects.count() == (0 if fail_second else 2) + assert set(BackgroundJob.objects.values_list("status", flat=True)) == ( + {"failed"} if fail_second else {"success"} + ) + + +@pytest.mark.django_db(transaction=True) +def test_cache_callback_failure_preserves_committed_success(run, caplog): + run, jobs, logs, organization = run + for source in jobs: + stage(run, source, organization) + ready(run, "industrial", jobs, logs) + with patch( + "apps.parsers.aggregate_publication.invalidate_source_data_cache", + side_effect=RuntimeError("cache unavailable"), + ), patch( + "apps.parsers.tasks_registry_snapshots._refresh_published_exports" + ) as export: + ready(run, "manufactures", jobs, logs) + export.assert_called_once() + assert set(BackgroundJob.objects.values_list("status", flat=True)) == {"success"} + assert set(ParserLoadLog.objects.values_list("status", flat=True)) == {"success"} + assert "cache invalidation failed" in caplog.text + + +def test_second_composed_refresh_is_rejected_without_extra_jobs(db): + from apps.core.exceptions import ConflictError + + definition = SourceCardService.get_definition("manufacturers-and-products") + with patch.object(SourceCardService, "_enqueue_task", return_value={}): + SourceCardService._launch_refresh(definition, requested_by_id=None, params={}) + with pytest.raises(ConflictError): + SourceCardService._launch_refresh( + definition, requested_by_id=None, params={} + ) + assert ParserRefreshRun.objects.count() == 1 + assert BackgroundJob.objects.count() == 3 diff --git a/tests/apps/parsers/test_refresh_progress.py b/tests/apps/parsers/test_refresh_progress.py index 43f5bbe..c3a6440 100644 --- a/tests/apps/parsers/test_refresh_progress.py +++ b/tests/apps/parsers/test_refresh_progress.py @@ -80,9 +80,9 @@ def test_failed_run_keeps_progress_and_error_after_last_active_job_disappears(): job.fail("Ошибка загрузки") card = SourceCardService.get_card("arbitration-cases") assert (card["status"], card["progress"], card["error_message"]) == ( - "error", + "failed", 60, - "Ошибка загрузки", + "Не удалось обновить данные. Повторите попытку позже.", ) @@ -110,13 +110,21 @@ def test_broker_failure_keeps_all_registered_parts_in_final_failed_state(): @pytest.mark.django_db def test_completed_new_run_does_not_hide_active_older_run(): definition = SourceCardService.get_definition("manufacturers-and-products") - specs = [(MagicMock(), name, "industrial") for name in definition.task_names] - first = SourceCardService._enqueue_refresh_group( - definition, specs, requested_by_id=None, kwargs={} - ) - second = SourceCardService._enqueue_refresh_group( - definition, specs, requested_by_id=None, kwargs={} - ) + + # Historical overlapping runs predate the publication barrier. New refreshes + # reject this overlap, but polling must still represent existing older jobs. + def legacy_run(prefix): + ids = [f"{prefix}-{index}" for index in range(len(definition.task_names))] + for task_id, name in zip(ids, definition.task_names, strict=True): + BackgroundJob.objects.create( + task_id=task_id, + task_name=name, + meta={"source_card": definition.slug, "refresh_task_ids": ids}, + ) + return [{"task_id": task_id} for task_id in ids] + + first = legacy_run("older") + second = legacy_run("newer") for item in second: BackgroundJob.objects.get(task_id=item["task_id"]).complete() card = SourceCardService.get_card(definition.slug) diff --git a/tests/apps/parsers/test_snapshot_public_contract.py b/tests/apps/parsers/test_snapshot_public_contract.py index 8da759b..e6602c7 100644 --- a/tests/apps/parsers/test_snapshot_public_contract.py +++ b/tests/apps/parsers/test_snapshot_public_contract.py @@ -234,7 +234,11 @@ def test_registry_job_status_distinguishes_queue_retry_failure_cancel( assert response.status_code == 200 assert response.data["status"] == expected assert response.data["progress"] == 47 - assert response.data["message"] == "fixture progress" + assert response.data["message"] == ( + "Не удалось обновить данные. Повторите попытку позже." + if expected in {"failed", "cancelled"} + else "fixture progress" + ) if raw == "success": job.result = {"status": "skipped"} job.save(update_fields=["result"]) @@ -267,9 +271,15 @@ def test_typed_refresh_job_dashboard_history_openapi(admin_client): ) assert paths[dashboard]["get"]["operationId"] == "api_v1_parsers_dashboard_list" assert paths[dashboard]["get"]["tags"] == ["api"] - assert {"pending", "started", "retry", "failure", "revoked"} <= set( - definitions["BackgroundJob"]["properties"]["status"]["enum"] - ) + assert { + "queued", + "running", + "retrying", + "failed", + "cancelled", + "success", + "skipped", + } == set(definitions["BackgroundJob"]["properties"]["status"]["enum"]) logs_path = next(path for path in paths if path.endswith("/system/logs/")) parameters = {item["name"]: item for item in paths[logs_path]["get"]["parameters"]} assert parameters["source"]["type"] == "string" diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index e1d0bce..6d946f3 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -428,7 +428,7 @@ class SourceCardServiceUnitTest(TestCase): latest_load=stale_in_progress_load, last_updated_at=None, ), - "error", + "failed", ) self.assertEqual( SourceCardService._get_status( @@ -437,7 +437,7 @@ class SourceCardServiceUnitTest(TestCase): latest_load=failed_load, last_updated_at=None, ), - "error", + "failed", ) self.assertEqual( SourceCardService._get_status( diff --git a/tests/apps/parsers/test_sources_api_e2e.py b/tests/apps/parsers/test_sources_api_e2e.py index 2552e47..af95630 100644 --- a/tests/apps/parsers/test_sources_api_e2e.py +++ b/tests/apps/parsers/test_sources_api_e2e.py @@ -125,9 +125,10 @@ class SourcesApiE2ETest(APITestCase): self.assertEqual(cards["financial-indicators"]["records_count"], 1) self.assertEqual(cards["planned-inspections"]["status"], "in_progress") self.assertEqual(cards["planned-inspections"]["progress"], 55) - self.assertEqual(cards["public-procurements"]["status"], "error") + self.assertEqual(cards["public-procurements"]["status"], "failed") self.assertEqual( - cards["public-procurements"]["error_message"], "download error" + cards["public-procurements"]["error_message"], + "Не удалось обновить данные. Повторите попытку позже.", ) minprom_card = detail_response.data["data"] @@ -142,7 +143,7 @@ class SourcesApiE2ETest(APITestCase): statuses = {item["slug"]: item for item in statuses_response.data["data"]} self.assertEqual(statuses["planned-inspections"]["progress"], 55) - self.assertEqual(statuses["public-procurements"]["status"], "error") + self.assertEqual(statuses["public-procurements"]["status"], "failed") def test_products_endpoint_supports_filter_and_search(self): target = IndustrialProductRecordFactory( diff --git a/tests/apps/parsers/test_system_logs_checkup.py b/tests/apps/parsers/test_system_logs_checkup.py index 41518fe..15d0c26 100644 --- a/tests/apps/parsers/test_system_logs_checkup.py +++ b/tests/apps/parsers/test_system_logs_checkup.py @@ -150,7 +150,7 @@ class SystemLogsCheckupTest(APITestCase): expected, ) - def test_csv_preserves_cyrillic_delimiters_quotes_and_newlines(self): + def test_csv_preserves_encoding_and_hides_internal_diagnostics(self): message = 'Ошибка; "детали"\r\nСледующая строка' self.create_log( datetime(2026, 9, 7, 12, 34, tzinfo=UTC), @@ -173,7 +173,11 @@ class SystemLogsCheckupTest(APITestCase): ) self.assertEqual(rows[1][1], "07.09.2026 12:34") self.assertEqual(rows[1][3], "Ошибка") - self.assertEqual(rows[1][rows[0].index("Ошибка")], message) + self.assertEqual( + rows[1][rows[0].index("Ошибка")], + "Не удалось обновить данные. Повторите попытку позже.", + ) + self.assertNotIn(message, decoded) def test_openapi_documents_period_for_list_and_export(self): response = self.client.get(reverse("schema-swagger-ui"), {"format": "openapi"}) diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 4f6c8ee..2a3a411 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -1894,6 +1894,17 @@ class MinpromtorgTasksTestCase(TestCase): "parse_fstec_registers", ) with ExitStack() as stack: + stack.enter_context( + patch.object( + parser_tasks, + "parse_all_minpromtorg", + return_value={ + "industrial": "industrial-id", + "industrial_products": "products-id", + "manufactures": "manufactures-id", + }, + ) + ) delays = { task_name: stack.enter_context( patch.object( @@ -1939,51 +1950,63 @@ class MinpromtorgTasksTestCase(TestCase): original_industrial = parser_tasks.IndustrialProductionClient original_industrial_products = parser_tasks.GispProductsClient original_manufactures = parser_tasks.ManufacturesClient - original_industrial_delay = parser_tasks.parse_industrial_production.delay - original_industrial_products_delay = ( - parser_tasks.parse_industrial_products.delay + original_industrial_delay = ( + parser_tasks.parse_industrial_production.apply_async ) - original_manufactures_delay = parser_tasks.parse_manufactures.delay + original_industrial_products_delay = ( + parser_tasks.parse_industrial_products.apply_async + ) + original_manufactures_delay = parser_tasks.parse_manufactures.apply_async parser_tasks.IndustrialProductionClient = _LocalIndustrialClient parser_tasks.GispProductsClient = _LocalIndustrialProductsClient parser_tasks.ManufacturesClient = _LocalManufacturesClient def _industrial_eager_delay(*args, **kwargs): return parser_tasks.parse_industrial_production.apply( - args=args, - kwargs=kwargs, + args=kwargs.get("args", args), + kwargs=kwargs.get("kwargs", {}), + task_id=kwargs.get("task_id"), + throw=True, ) def _industrial_products_eager_delay(*args, **kwargs): return parser_tasks.parse_industrial_products.apply( - args=args, - kwargs=kwargs, + args=kwargs.get("args", args), + kwargs=kwargs.get("kwargs", {}), + task_id=kwargs.get("task_id"), + throw=True, ) def _manufactures_eager_delay(*args, **kwargs): return parser_tasks.parse_manufactures.apply( - args=args, - kwargs=kwargs, + args=kwargs.get("args", args), + kwargs=kwargs.get("kwargs", {}), + task_id=kwargs.get("task_id"), + throw=True, ) - parser_tasks.parse_industrial_production.delay = _industrial_eager_delay - parser_tasks.parse_industrial_products.delay = ( + parser_tasks.parse_industrial_production.apply_async = ( + _industrial_eager_delay + ) + parser_tasks.parse_industrial_products.apply_async = ( _industrial_products_eager_delay ) - parser_tasks.parse_manufactures.delay = _manufactures_eager_delay + parser_tasks.parse_manufactures.apply_async = _manufactures_eager_delay try: result = parse_all_minpromtorg(proxies=[]) finally: parser_tasks.IndustrialProductionClient = original_industrial parser_tasks.GispProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures - parser_tasks.parse_industrial_production.delay = ( + parser_tasks.parse_industrial_production.apply_async = ( original_industrial_delay ) - parser_tasks.parse_industrial_products.delay = ( + parser_tasks.parse_industrial_products.apply_async = ( original_industrial_products_delay ) - parser_tasks.parse_manufactures.delay = original_manufactures_delay + parser_tasks.parse_manufactures.apply_async = ( + original_manufactures_delay + ) self.assertIn("industrial", result) self.assertIn("industrial_products", result) @@ -2029,11 +2052,13 @@ class MinpromtorgTasksTestCase(TestCase): original_industrial = parser_tasks.IndustrialProductionClient original_industrial_products = parser_tasks.GispProductsClient original_manufactures = parser_tasks.ManufacturesClient - original_industrial_delay = parser_tasks.parse_industrial_production.delay - original_industrial_products_delay = ( - parser_tasks.parse_industrial_products.delay + original_industrial_delay = ( + parser_tasks.parse_industrial_production.apply_async ) - original_manufactures_delay = parser_tasks.parse_manufactures.delay + original_industrial_products_delay = ( + parser_tasks.parse_industrial_products.apply_async + ) + original_manufactures_delay = parser_tasks.parse_manufactures.apply_async original_sync_inspections_delay = parser_tasks.sync_inspections.delay parser_tasks.IndustrialProductionClient = _LocalIndustrialClient parser_tasks.GispProductsClient = _LocalIndustrialProductsClient @@ -2041,30 +2066,38 @@ class MinpromtorgTasksTestCase(TestCase): def _industrial_eager_delay(*args, **kwargs): return parser_tasks.parse_industrial_production.apply( - args=args, - kwargs=kwargs, + args=kwargs.get("args", args), + kwargs=kwargs.get("kwargs", {}), + task_id=kwargs.get("task_id"), + throw=True, ) def _industrial_products_eager_delay(*args, **kwargs): return parser_tasks.parse_industrial_products.apply( - args=args, - kwargs=kwargs, + args=kwargs.get("args", args), + kwargs=kwargs.get("kwargs", {}), + task_id=kwargs.get("task_id"), + throw=True, ) def _manufactures_eager_delay(*args, **kwargs): return parser_tasks.parse_manufactures.apply( - args=args, - kwargs=kwargs, + args=kwargs.get("args", args), + kwargs=kwargs.get("kwargs", {}), + task_id=kwargs.get("task_id"), + throw=True, ) def _inspections_stub_delay(*_args, **_kwargs): return SimpleNamespace(id="inspections-test-task") - parser_tasks.parse_industrial_production.delay = _industrial_eager_delay - parser_tasks.parse_industrial_products.delay = ( + parser_tasks.parse_industrial_production.apply_async = ( + _industrial_eager_delay + ) + parser_tasks.parse_industrial_products.apply_async = ( _industrial_products_eager_delay ) - parser_tasks.parse_manufactures.delay = _manufactures_eager_delay + parser_tasks.parse_manufactures.apply_async = _manufactures_eager_delay parser_tasks.sync_inspections.delay = _inspections_stub_delay try: result = parse_all_sources(proxies=[], inspections_use_playwright=None) @@ -2072,13 +2105,15 @@ class MinpromtorgTasksTestCase(TestCase): parser_tasks.IndustrialProductionClient = original_industrial parser_tasks.GispProductsClient = original_industrial_products parser_tasks.ManufacturesClient = original_manufactures - parser_tasks.parse_industrial_production.delay = ( + parser_tasks.parse_industrial_production.apply_async = ( original_industrial_delay ) - parser_tasks.parse_industrial_products.delay = ( + parser_tasks.parse_industrial_products.apply_async = ( original_industrial_products_delay ) - parser_tasks.parse_manufactures.delay = original_manufactures_delay + parser_tasks.parse_manufactures.apply_async = ( + original_manufactures_delay + ) parser_tasks.sync_inspections.delay = original_sync_inspections_delay self.assertIn("industrial", result) diff --git a/tools/openapi/build.mjs b/tools/openapi/build.mjs index 3ce3557..c20b2a3 100644 --- a/tools/openapi/build.mjs +++ b/tools/openapi/build.mjs @@ -172,6 +172,19 @@ for (const [prefix, group, source, recordType, payloadPrefix] of extendedVariant } } +// Swagger 2 produces is operation-wide; binary success and JSON errors differ. +for (const endpoint of ['export', 'export-download']) { + const operation = document.paths[`/api/v2/organization-source-records/${endpoint}/`]?.post; + if (!operation) throw new Error(`Source export endpoint missing: ${endpoint}`); + operation.responses['200'].content = { + 'application/zip': { schema: { type: 'string', format: 'binary' } }, + }; + for (const [code, response] of Object.entries(operation.responses)) { + if (Number(code) < 400 || !response.content) continue; + response.content = { 'application/json': Object.values(response.content)[0] }; + } +} + const attachmentPath = document.paths['/api/v2/organization-source-records/{uid}/documents/{document_id}/attachment/']; if (!attachmentPath?.get) throw new Error('Integrated attachment endpoint missing'); attachmentPath.get.responses['200'] = { diff --git a/tools/openapi/check-runtime.mjs b/tools/openapi/check-runtime.mjs index 7930b63..c5ea267 100644 --- a/tools/openapi/check-runtime.mjs +++ b/tools/openapi/check-runtime.mjs @@ -1,6 +1,9 @@ import { readFile } from 'node:fs/promises'; import { isDeepStrictEqual } from 'node:util'; import { + V2OrganizationSourceRecordsFilterOptions200Response, + V2OrganizationSourceRecordsFilterOptions400Response, + ApiV1JobsRead200Response, ApiV2OrganizationSourceRecordsRead200Response, ApiV2OrganizationSourceRecordsRead400Response, ApiV2OrganizationSourceRecordsRead404Response, @@ -14,6 +17,9 @@ import { } from './generated-zod/contract.ts'; const schemas = new Map([ + ['options:200', V2OrganizationSourceRecordsFilterOptions200Response], + ['options:400', V2OrganizationSourceRecordsFilterOptions400Response], + ['job:200', ApiV1JobsRead200Response], ['refresh:202', ApiV1SourcesRefreshCreate202Response], ['refresh:400', ApiV1SourcesRefreshCreate400Response], ['refresh:403', ApiV1SourcesRefreshCreate403Response], @@ -26,6 +32,7 @@ const schemas = new Map([ ['detail:404', ApiV2OrganizationSourceRecordsRead404Response], ]); const requiredCases = new Set([ + 'message-type-options', 'message-type-options-invalid', 'job-failed', 'job-success', 'job-skipped', 'mixed-list', 'empty-list', 'invalid-list', 'invalid-detail', 'missing-detail', 'stock_exchange_activity-spb-detail', 'missing-refresh', ...['fssp_enforcement_proceedings', 'checko_leasing', 'checko_messages', 'checko_licenses', 'stock_exchange_activity']