diff --git a/docker/Dockerfile b/docker/Dockerfile index 48d17dc..eb36b12 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -91,7 +91,7 @@ ENV PATH="/app/.venv/bin:${PATH}" \ CELERY_RESULT_BACKEND=redis://redis:6379/0 \ PORT=8000 \ GUNICORN_WORKERS=4 \ - GUNICORN_TIMEOUT=60 \ + GUNICORN_TIMEOUT=300 \ CELERY_LOG_LEVEL=INFO \ CELERY_WORKER_CONCURRENCY=2 \ CHECKO_API_KEY= \ diff --git a/docker/scripts/start-web.sh b/docker/scripts/start-web.sh index 32a1daf..7aeeb55 100755 --- a/docker/scripts/start-web.sh +++ b/docker/scripts/start-web.sh @@ -22,6 +22,6 @@ python src/manage.py migrate --noinput exec gunicorn core.wsgi:application \ --bind "0.0.0.0:${PORT:-8000}" \ --workers "${GUNICORN_WORKERS:-3}" \ - --timeout "${GUNICORN_TIMEOUT:-60}" \ + --timeout "${GUNICORN_TIMEOUT:-300}" \ --access-logfile "-" \ --error-logfile "-" diff --git a/src/apps/backups/views.py b/src/apps/backups/views.py index 3ec146a..89d58ef 100644 --- a/src/apps/backups/views.py +++ b/src/apps/backups/views.py @@ -1,8 +1,11 @@ """API views экспорта защищённых backup архивов.""" from apps.backups.serializers import BackupExportRequestSerializer -from apps.backups.services import BackupExportError, BackupExportJobService from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag +from apps.exchange.state_corp_services import ( + StateCorpExchangeError, + StateCorpExchangeService, +) from django.http import HttpResponse from django.utils import timezone from drf_yasg import openapi @@ -10,14 +13,13 @@ from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAdminUser -from rest_framework.response import Response from rest_framework.views import APIView BACKUPS_TAG = swagger_tag("Резервные копии", "backups") class BackupExportView(APIView): - """Асинхронный экспорт защищённого backup архива.""" + """Синхронный экспорт защищённого ZIP-пакета для state-corp.""" permission_classes = [IsAdminUser] @@ -25,16 +27,12 @@ class BackupExportView(APIView): tags=[BACKUPS_TAG], operation_summary="Сформировать backup архив", operation_description=( - "Асинхронное формирование архива `.zip` на дату актуальности.\n" - "Логика endpoint:\n" - "- если задача на дату уже выполняется: вернуть 'подождите'\n" - "- если задачи нет: запустить формирование\n" - "- если задача завершена успешно: отдать архив и удалить его\n\n" + "Синхронное формирование архива `.zip` на дату актуальности.\n" "Внутри архива:\n" - "- бинарный зашифрованный backup `.bin`\n" + "- бинарный зашифрованный пакет `.bin`\n" "- контрольная сумма `.sha256`\n\n" - "Экспортирует только актуальные организации из реестров на указанную дату " - "и все связанные записи из parser-таблиц." + "Экспортирует организации из реестров Росатом и Роскосмос на указанную " + "дату и связанные записи из parser-таблиц в контракте импорта state-corp." ), request_body=BackupExportRequestSerializer, responses={ @@ -42,21 +40,6 @@ class BackupExportView(APIView): description="Готовый backup-архив (application/zip)", schema=openapi.Schema(type=openapi.TYPE_FILE), ), - 202: openapi.Response( - description="Задача запущена или выполняется", - schema=openapi.Schema( - type=openapi.TYPE_OBJECT, - properties={ - "status": openapi.Schema(type=openapi.TYPE_STRING), - "message": openapi.Schema(type=openapi.TYPE_STRING), - "actual_date": openapi.Schema( - type=openapi.TYPE_STRING, - format=openapi.FORMAT_DATE, - ), - "task_id": openapi.Schema(type=openapi.TYPE_STRING), - }, - ), - ), 400: CommonResponses.BAD_REQUEST, **ErrorResponses.ADMIN, }, @@ -69,40 +52,23 @@ class BackupExportView(APIView): ) try: - result = BackupExportJobService.check_or_start_job( - actual_date=actual_date, - requested_by_id=request.user.id - if request.user.is_authenticated - else None, - ) - except BackupExportError as exc: + package = StateCorpExchangeService.build_package(actual_date=actual_date) + except StateCorpExchangeError as exc: raise ValidationError({"backup": str(exc)}) from exc - if result.action in {"started", "wait"}: - return Response( - { - "status": result.action, - "message": result.message, - "actual_date": result.actual_date.isoformat(), - "task_id": result.task_id, - }, - status=status.HTTP_202_ACCEPTED, - ) - - try: - artifact = BackupExportJobService.consume_ready_archive( - actual_date=result.actual_date - ) - except BackupExportError as exc: - raise ValidationError({"backup": str(exc)}) from exc - - response = HttpResponse(artifact.archive_bytes, content_type="application/zip") + response = HttpResponse(package.archive_bytes, content_type="application/zip") response.status_code = status.HTTP_200_OK response[ "Content-Disposition" - ] = f'attachment; filename="{artifact.archive_filename}"' - response["X-Backup-SHA256"] = artifact.checksum_sha256 - response["X-Backup-Checksum-File"] = artifact.checksum_filename - response["X-Backup-Organizations"] = str(artifact.organizations_count) - response["X-Backup-Actual-Date"] = artifact.actual_date.isoformat() + ] = f'attachment; filename="{package.archive_name}"' + response["Content-Length"] = str(len(package.archive_bytes)) + response["X-State-Corp-Package-Id"] = package.package_id + response["X-State-Corp-Bin-File"] = package.bin_name + response["X-State-Corp-Organizations"] = str( + package.payload_counts.get("organizations", 0) + ) + response["X-Backup-Organizations"] = str( + package.payload_counts.get("organizations", 0) + ) + response["X-Backup-Actual-Date"] = actual_date.isoformat() return response diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index 9df89c1..9533032 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -19,14 +19,17 @@ from zipfile import ZIP_DEFLATED, ZipFile import requests from apps.parsers.models import ( + GenericParserRecord, IndustrialProductRecord, InspectionRecord, + ParserLoadLog, ProcurementRecord, ) from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings +from django.db.models import Q from django.utils import timezone -from registers.models import Organization +from registers.models import Organization, RegistryMembershipPeriod class StateCorpExchangeError(ValueError): @@ -52,6 +55,14 @@ class StateCorpExchangeService: AAD = b"state-corp-exchange-v1" PAYLOAD_FORMAT = "state-corp-exchange-payload" BIN_FORMAT = "state-corp-exchange-bin" + ROSATOM_ROSCOSMOS_REGISTRY_NAMES = ( + "Реестр госкорпорации Роскосмос", + "Реестр госкорпорации Роскосмос ГОЗ", + "Реестр госкорпорации Роскосмос ОПК", + "Реестр госкорпорации Росатом", + "Реестр госкорпорации Росатом ГОЗ", + "Реестр госкорпорации Росатом ОПК", + ) @classmethod def build_package( @@ -60,18 +71,32 @@ class StateCorpExchangeService: package_id: str | None = None, source_system: str = "mostovik", organization_inns: list[str] | None = None, + actual_date: date | str | None = None, ) -> StateCorpExchangePackage: """Build encrypted archive for state-corp import.""" produced_at = timezone.now() + snapshot_date = cls._coerce_actual_date(actual_date) normalized_inns = cls._normalize_inn_list(organization_inns) - organizations = cls._get_organizations(normalized_inns) + organizations = ( + cls._get_organizations(normalized_inns) + if normalized_inns + else cls._get_rosatom_roscosmos_organizations(snapshot_date) + ) allowed_inns = {str(item.mn_inn) for item in organizations} data = { "organizations": cls._serialize_organizations(organizations), "industrial_products": cls._serialize_industrial_products(allowed_inns), "prosecutor_checks": cls._serialize_prosecutor_checks(allowed_inns), "public_procurements": cls._serialize_public_procurements(allowed_inns), - "arbitration_cases": [], + "arbitration_cases": cls._serialize_arbitration_cases(allowed_inns), + "bankruptcy_procedures": cls._serialize_bankruptcy_procedures(allowed_inns), + "defense_unreliable_suppliers": ( + cls._serialize_defense_unreliable_suppliers(allowed_inns) + ), + "information_security_registries": ( + cls._serialize_information_security_registries(allowed_inns) + ), + "labor_vacancies": cls._serialize_labor_vacancies(allowed_inns), } payload_counts = {key: len(value) for key, value in data.items()} package_id = package_id or cls._build_package_id() @@ -248,6 +273,23 @@ class StateCorpExchangeService: ) return list(queryset) + @classmethod + def _get_rosatom_roscosmos_organizations( + cls, + actual_date: date, + ) -> list[Organization]: + organization_ids = ( + RegistryMembershipPeriod.objects.filter( + registry__name__in=cls.ROSATOM_ROSCOSMOS_REGISTRY_NAMES, + started_at__lte=actual_date, + ) + .filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date)) + .order_by("organization_id") + .values_list("organization_id", flat=True) + .distinct() + ) + return list(Organization.objects.filter(id__in=organization_ids).order_by("id")) + @classmethod def _serialize_organizations( cls, @@ -350,6 +392,249 @@ class StateCorpExchangeService: ) return items + @classmethod + def _serialize_arbitration_cases( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str]]: + items: list[dict[str, str]] = [] + queryset = GenericParserRecord.objects.filter( + source=ParserLoadLog.Source.ARBITRATION, + inn__in=allowed_inns, + ).order_by("id") + for record in queryset: + payload = record.payload if isinstance(record.payload, dict) else {} + target = payload.get("target") + if not isinstance(target, dict): + target = {} + + decision_date = cls._coerce_date( + None, + payload.get("result_date") + or payload.get("decision_date") + or payload.get("filing_date") + or record.record_date, + ) + case_number = str( + payload.get("case_number") or record.title or record.external_id or "" + ).strip() + if not case_number or decision_date is None: + continue + + items.append( + { + "organization_inn": cls._digits(record.inn), + "case_number": case_number, + "court_name": str(payload.get("court_name") or "").strip(), + "party_role": str( + target.get("role") or payload.get("party_role") or "" + ).strip(), + "status": str(payload.get("status") or record.status or "").strip(), + "decision_date": decision_date.isoformat(), + } + ) + return items + + @classmethod + def _serialize_bankruptcy_procedures( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str | None]]: + items: list[dict[str, str | None]] = [] + for record in cls._generic_records( + allowed_inns, + sources=[ParserLoadLog.Source.FEDRESURS_BANKRUPTCY], + ): + payload = cls._record_payload(record) + message_date = cls._coerce_date( + None, + cls._payload_lookup(payload, ["message_date", "date", "Дата"]) + or record.record_date, + ) + message_type = cls._payload_lookup(payload, ["message_type", "type"]) + items.append( + { + "organization_inn": cls._digits(record.inn), + "external_id": record.external_id, + "message_type": message_type or record.title or record.status, + "message_date": message_date.isoformat() if message_date else None, + "case_number": cls._payload_lookup(payload, ["case_number"]), + "status": record.status, + "source_url": record.url, + } + ) + return items + + @classmethod + def _serialize_defense_unreliable_suppliers( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str | None]]: + items: list[dict[str, str | None]] = [] + for record in cls._generic_records( + allowed_inns, + sources=[ + ParserLoadLog.Source.UNFAIR_SUPPLIERS, + ParserLoadLog.Source.FAS_GOZ, + ], + ): + payload = cls._record_payload(record) + included_at = cls._coerce_date( + None, + cls._payload_lookup( + payload, + [ + "included_at", + "date", + "Дата вступления постановления", + "Включено", + ], + ) + or record.record_date, + ) + items.append( + { + "organization_inn": cls._digits(record.inn), + "external_id": record.external_id, + "registry_source": record.source, + "registry_number": cls._payload_lookup( + payload, + [ + "registry_number", + "number", + "Номер реестровой записи", + "Номер реестровой записи в ЕРУЗ", + ], + ), + "supplier_name": cls._payload_lookup( + payload, + [ + "supplier_name", + "organisation_name", + "Полное наименование лица", + "Фирменное наименование лица", + "Наименование ФИО недобросовестного поставщика", + ], + ) + or record.organisation_name, + "reason": record.title, + "included_at": included_at.isoformat() if included_at else None, + "status": record.status, + "source_url": record.url, + } + ) + return items + + @classmethod + def _serialize_information_security_registries( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str | None]]: + items: list[dict[str, str | None]] = [] + for record in cls._generic_records( + allowed_inns, + sources=[ParserLoadLog.Source.FSTEC], + ): + payload = cls._record_payload(record) + issued_at = cls._coerce_date( + None, + cls._payload_lookup( + payload, + ["issued_at", "date", "Дата предоставления лицензии"], + ) + or record.record_date, + ) + expires_at = cls._coerce_date( + None, + cls._payload_lookup( + payload, + ["expires_at", "Срок действия сертификата", "res_valid_till"], + ), + ) + items.append( + { + "organization_inn": cls._digits(record.inn), + "external_id": record.external_id, + "registry_name": cls._payload_lookup(payload, ["registry_name"]) + or record.title + or "Реестр ФСТЭК", + "presence_status": "present", + "entry_number": cls._payload_lookup( + payload, + [ + "entry_number", + "registration_number", + "Регистрационный номер лицензии", + "№ сертификата", + ], + ) + or record.external_id, + "issued_at": issued_at.isoformat() if issued_at else None, + "expires_at": expires_at.isoformat() if expires_at else None, + } + ) + return items + + @classmethod + def _serialize_labor_vacancies( + cls, + allowed_inns: set[str], + ) -> list[dict[str, str | None]]: + items: list[dict[str, str | None]] = [] + for record in cls._generic_records( + allowed_inns, + sources=[ParserLoadLog.Source.TRUDVSEM], + ): + payload = cls._record_payload(record) + published_at = cls._coerce_date( + None, + cls._payload_lookup( + payload, + ["published_at", "creation-date", "date", "published_at"], + ) + or record.record_date, + ) + items.append( + { + "organization_inn": cls._digits(record.inn), + "external_id": record.external_id, + "vacancy_source": cls._payload_lookup(payload, ["vacancy_source"]) + or "trudvsem", + "title": record.title, + "status": record.status, + "published_at": published_at.isoformat() if published_at else None, + "salary_amount": cls._serialize_decimal(record.amount), + "source_url": record.url, + } + ) + return items + + @staticmethod + def _record_payload(record: GenericParserRecord) -> dict[str, Any]: + return record.payload if isinstance(record.payload, dict) else {} + + @classmethod + def _generic_records( + cls, + allowed_inns: set[str], + *, + sources: list[str], + ): + if not allowed_inns: + return GenericParserRecord.objects.none() + return GenericParserRecord.objects.filter( + source__in=sources, + inn__in=allowed_inns, + ).order_by("id") + + @staticmethod + def _payload_lookup(payload: dict[str, Any], candidates: list[str]) -> str: + for key in candidates: + value = payload.get(key) + if value not in (None, ""): + return str(value).strip() + return "" + @staticmethod def _normalize_inn_list(organization_inns: list[str] | None) -> list[str]: normalized_items: list[str] = [] @@ -359,6 +644,20 @@ class StateCorpExchangeService: normalized_items.append(normalized) return normalized_items + @staticmethod + def _coerce_actual_date(value: date | str | None) -> date: + if value is None: + return timezone.localdate() + if isinstance(value, date): + return value + raw_text = str(value).strip() + try: + return date.fromisoformat(raw_text) + except ValueError as exc: + raise StateCorpExchangeError( + "actual_date должен быть датой в формате YYYY-MM-DD" + ) from exc + @staticmethod def _digits(value: str | int) -> str: return "".join(char for char in str(value).strip() if char.isdigit()) diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index ed14d33..33e65c0 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -218,6 +218,94 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( ), ), ), + SourceCardDefinition( + slug="bankruptcy-procedures", + title="Сведения о процедурах банкротства", + description="Сведения Федресурса о процедурах банкротства.", + order=50, + task_names=("apps.parsers.tasks.parse_fedresurs_bankruptcy",), + source_items=( + SourceItemDefinition( + code="fedresurs_bankruptcy", + title="Банкротства Федресурс", + description="Сведения о процедурах банкротства из ЕФРСБ.", + parser_source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY, + ), + ), + refresh_interval=timedelta(days=1), + ), + SourceCardDefinition( + slug="defense-unreliable-suppliers", + title="Недобросовестные поставщики ГОЗ", + description="Реестры недобросовестных поставщиков и уклонения от ГОЗ.", + order=60, + task_names=( + "apps.parsers.tasks.parse_unfair_suppliers", + "apps.parsers.tasks.parse_fas_goz_evasion", + ), + source_items=( + SourceItemDefinition( + code="unfair_suppliers", + title="Недобросовестные поставщики", + description="Реестр недобросовестных поставщиков ФАС и ЕИС.", + parser_source=ParserLoadLog.Source.UNFAIR_SUPPLIERS, + ), + SourceItemDefinition( + code="fas_goz", + title="Уклонение от ГОЗ", + description="Юрлица, привлеченные за отказ или уклонение от ГОЗ.", + parser_source=ParserLoadLog.Source.FAS_GOZ, + ), + ), + refresh_interval=timedelta(days=1), + ), + SourceCardDefinition( + slug="arbitration-cases", + title="Арбитражные дела", + description="Арбитражные дела по организациям.", + order=70, + task_names=("apps.parsers.tasks.parse_arbitration_cases",), + source_items=( + SourceItemDefinition( + code="arbitration", + title="Арбитражные дела", + description="Карточки арбитражных дел по ИНН/ОГРН организаций.", + parser_source=ParserLoadLog.Source.ARBITRATION, + ), + ), + refresh_interval=timedelta(days=1), + ), + SourceCardDefinition( + slug="information-security-registries", + title="Реестры по информационной безопасности", + description="Реестры ФСТЭК по информационной безопасности.", + order=80, + task_names=("apps.parsers.tasks.parse_fstec_registers",), + source_items=( + SourceItemDefinition( + code="fstec", + title="Реестры ФСТЭК", + description="Официальные реестры ФСТЭК по информационной безопасности.", + parser_source=ParserLoadLog.Source.FSTEC, + ), + ), + refresh_interval=timedelta(days=30), + ), + SourceCardDefinition( + slug="labor-vacancies", + title="Вакансии Работа России", + description="Вакансии работодателей, состоящих в активных реестрах.", + order=90, + task_names=("apps.parsers.tasks.parse_trudvsem_vacancies",), + source_items=( + SourceItemDefinition( + code="trudvsem", + title="Вакансии Работа России", + description="Вакансии работодателей из Работа России, HH и SuperJob.", + parser_source=ParserLoadLog.Source.TRUDVSEM, + ), + ), + ), ) SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS} @@ -231,6 +319,12 @@ GENERIC_RECORD_SOURCES_BY_ITEM_CODE = { "procurements_44fz": ParserLoadLog.Source.PROCUREMENTS_44FZ, "procurements_223fz": ParserLoadLog.Source.PROCUREMENTS_223FZ, "contracts": ParserLoadLog.Source.CONTRACTS, + "unfair_suppliers": ParserLoadLog.Source.UNFAIR_SUPPLIERS, + "fas_goz": ParserLoadLog.Source.FAS_GOZ, + "arbitration": ParserLoadLog.Source.ARBITRATION, + "fedresurs_bankruptcy": ParserLoadLog.Source.FEDRESURS_BANKRUPTCY, + "fstec": ParserLoadLog.Source.FSTEC, + "trudvsem": ParserLoadLog.Source.TRUDVSEM, } @@ -545,8 +639,81 @@ class SourceCardService: ) return [task_info] + task_specs = cls._get_simple_refresh_task_specs(definition.slug) + if task_specs: + return [ + cls._enqueue_task( + task=task, + task_name=task_name, + requested_by_id=requested_by_id, + meta={ + "source_card": definition.slug, + "source": parser_source, + }, + kwargs={"requested_by_id": requested_by_id}, + ) + for task, task_name, parser_source in task_specs + ] + raise ValidationError({"detail": "Обновление для карточки не поддерживается."}) + @staticmethod + def _get_simple_refresh_task_specs( + slug: str, + ) -> tuple[tuple[Any, str, str], ...]: + from apps.parsers.tasks import ( + parse_arbitration_cases, + parse_fas_goz_evasion, + parse_fedresurs_bankruptcy, + parse_fstec_registers, + parse_trudvsem_vacancies, + parse_unfair_suppliers, + ) + + specs = { + "bankruptcy-procedures": ( + ( + parse_fedresurs_bankruptcy, + "apps.parsers.tasks.parse_fedresurs_bankruptcy", + ParserLoadLog.Source.FEDRESURS_BANKRUPTCY, + ), + ), + "defense-unreliable-suppliers": ( + ( + parse_unfair_suppliers, + "apps.parsers.tasks.parse_unfair_suppliers", + ParserLoadLog.Source.UNFAIR_SUPPLIERS, + ), + ( + parse_fas_goz_evasion, + "apps.parsers.tasks.parse_fas_goz_evasion", + ParserLoadLog.Source.FAS_GOZ, + ), + ), + "arbitration-cases": ( + ( + parse_arbitration_cases, + "apps.parsers.tasks.parse_arbitration_cases", + ParserLoadLog.Source.ARBITRATION, + ), + ), + "information-security-registries": ( + ( + parse_fstec_registers, + "apps.parsers.tasks.parse_fstec_registers", + ParserLoadLog.Source.FSTEC, + ), + ), + "labor-vacancies": ( + ( + parse_trudvsem_vacancies, + "apps.parsers.tasks.parse_trudvsem_vacancies", + ParserLoadLog.Source.TRUDVSEM, + ), + ), + } + return specs.get(slug, ()) + @classmethod def _enqueue_task( cls, @@ -717,6 +884,7 @@ class SourceCardService: source_items: list[dict[str, Any]], ) -> int: if definition.slug == "public-procurements": + generic_sources = cls._get_generic_sources_for_definition(definition) legacy_inns = ( ProcurementRecord.objects.exclude(customer_inn="") .order_by() @@ -724,9 +892,7 @@ class SourceCardService: .distinct() ) generic_inns = ( - GenericParserRecord.objects.filter( - source__in=GENERIC_RECORD_SOURCES_BY_ITEM_CODE.values() - ) + GenericParserRecord.objects.filter(source__in=generic_sources) .exclude(inn="") .order_by() .values_list("inn", flat=True) @@ -735,6 +901,15 @@ class SourceCardService: return legacy_inns.union(generic_inns).count() if definition.slug != "manufacturers-and-products": + generic_sources = cls._get_generic_sources_for_definition(definition) + if generic_sources and len(generic_sources) == len(definition.source_items): + return ( + GenericParserRecord.objects.filter(source__in=generic_sources) + .exclude(inn="") + .values("inn") + .distinct() + .count() + ) return sum(item["organizations_count"] for item in source_items) industrial_inns = ( @@ -757,6 +932,18 @@ class SourceCardService: ) return industrial_inns.union(manufacturer_inns, product_inns).count() + @staticmethod + def _get_generic_sources_for_definition( + definition: SourceCardDefinition, + ) -> list[str]: + return list( + dict.fromkeys( + GENERIC_RECORD_SOURCES_BY_ITEM_CODE[item.code] + for item in definition.source_items + if item.code in GENERIC_RECORD_SOURCES_BY_ITEM_CODE + ) + ) + @classmethod def _get_latest_load( cls, diff --git a/src/templates/dashboard.html b/src/templates/dashboard.html index fb09966..37e1d21 100644 --- a/src/templates/dashboard.html +++ b/src/templates/dashboard.html @@ -1847,8 +1847,8 @@
-

Выгрузка организаций реестра в .bin

-

Формирование защищенного архива по выбранному реестру и дате актуальности.

+

Выгрузка организаций Росатом/Роскосмос в ZIP

+

Синхронное формирование защищенного ZIP-архива для импорта в state-corp.

@@ -1857,11 +1857,11 @@
- + Запрос уйдет в /api/v1/backups/export/
-
Выберите реестр и дату для защищенной выгрузки.
+
Выберите дату для защищенной выгрузки.
@@ -2914,7 +2914,7 @@ ["Registers", "Registries", "GET", "/api/v1/registers/registries/?page_size=1"], ["Registers", "Organizations", "GET", "/api/v1/registers/organizations/?page_size=1"], ["Registers", "Upload registry", "POST", "/api/v2/registers/opk/upload/"], - ["Backups", "Export .bin", "POST", "/api/v1/backups/export/"], + ["Backups", "Export state-corp ZIP", "POST", "/api/v1/backups/export/"], ["Exchange", "Connections", "GET", "/api/v1/exchange/connections/"], ["Exchange", "Create connection", "POST", "/api/v1/exchange/connections/"], ["Exchange", "Test connection", "POST", "/api/v1/exchange/connections/test/"], @@ -4303,7 +4303,7 @@ $("organizationRegistryFilter").innerHTML = ``; renderOrganizationDataSourceOptions(); $("registrySummary").innerHTML = `
Реестры недоступны: ${escapeHtml(errorMessage(error))}
`; - $("backupStatus").textContent = `Выгрузка .bin недоступна: ${errorMessage(error)}`; + $("backupStatus").textContent = `Выгрузка ZIP недоступна: ${errorMessage(error)}`; } function setSelectOptions(selectId, tables) { @@ -4899,7 +4899,7 @@ return; } const payload = result.data || result; - $("backupStatus").textContent = `${payload.status || "queued"}: ${payload.message || ""} task_id=${payload.task_id || ""}`; + $("backupStatus").textContent = `${payload.status || "done"}: ${payload.message || ""}`; await refreshDashboard(); } catch (error) { $("backupStatus").textContent = JSON.stringify(error); diff --git a/tests/apps/backups/test_views.py b/tests/apps/backups/test_views.py index 00e2fb8..fb9f9e3 100644 --- a/tests/apps/backups/test_views.py +++ b/tests/apps/backups/test_views.py @@ -1,21 +1,13 @@ -"""Tests for async backups export API.""" +"""Tests for backups export API.""" from __future__ import annotations -import hashlib -from pathlib import Path -from tempfile import TemporaryDirectory +from datetime import date +from types import SimpleNamespace from unittest.mock import patch -from apps.backups.models import BackupExportJob -from apps.backups.services import ( - BackupExportError, - BackupExportJobService, - BackupRequestResult, -) -from django.db import IntegrityError +from apps.exchange.state_corp_services import StateCorpExchangeError from django.urls import reverse -from django.utils import timezone from rest_framework import status from rest_framework.test import APITestCase @@ -23,7 +15,7 @@ from tests.apps.user.factories import UserFactory class BackupExportViewTest(APITestCase): - """Tests for async backup export endpoint.""" + """Tests for synchronous ZIP export endpoint.""" def setUp(self): self.user = UserFactory.create_user() @@ -38,167 +30,49 @@ class BackupExportViewTest(APITestCase): response = self.client.post(self.url, {}, format="json") self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") - def test_export_starts_job_when_absent(self, enqueue_mock): - self.client.force_authenticate(self.admin) - today = timezone.localdate() - with self.captureOnCommitCallbacks(execute=True): - response = self.client.post( - self.url, - {"actual_date": today.isoformat()}, - format="json", - ) - - self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - self.assertEqual(response.data["status"], "started") - - job = BackupExportJob.objects.get(actual_date=today) - self.assertEqual(job.status, BackupExportJob.Status.PENDING) - self.assertEqual(response.data["task_id"], job.task_id) - enqueue_mock.assert_called_once_with(job_id=job.id, task_id=job.task_id) - - @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") - def test_export_returns_wait_when_job_in_progress(self, enqueue_mock): - today = timezone.localdate() - BackupExportJob.objects.create( - actual_date=today, - status=BackupExportJob.Status.STARTED, - task_id="task-running-1", + @patch("apps.backups.views.StateCorpExchangeService.build_package") + def test_export_returns_state_corp_zip_synchronously(self, build_package_mock): + build_package_mock.return_value = SimpleNamespace( + package_id="package-1", + archive_name="state_corp_exchange.zip", + bin_name="state_corp_exchange.bin", + archive_bytes=b"zip-bytes", + payload_counts={ + "organizations": 2, + "industrial_products": 1, + "prosecutor_checks": 0, + "public_procurements": 0, + "arbitration_cases": 0, + }, + produced_at="2026-05-12T10:00:00+00:00", ) self.client.force_authenticate(self.admin) response = self.client.post( self.url, - {"actual_date": today.isoformat()}, + {"actual_date": "2026-05-12", "registry": "ignored-legacy-field"}, format="json", ) - self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - self.assertEqual(response.data["status"], "wait") - self.assertEqual(response.data["task_id"], "task-running-1") - enqueue_mock.assert_not_called() - - @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") - def test_export_returns_file_and_deletes_after_download(self, enqueue_mock): - with TemporaryDirectory() as tmp_dir: - tmp_path = Path(tmp_dir) - archive_bytes = b"zip-content" - archive_path = tmp_path / "backup.zip" - archive_path.write_bytes(archive_bytes) - - today = timezone.localdate() - job = BackupExportJob.objects.create( - actual_date=today, - status=BackupExportJob.Status.SUCCESS, - task_id="task-success-1", - archive_path=str(archive_path), - archive_filename="backup.zip", - checksum_filename="backup.zip.sha256", - checksum_sha256=hashlib.sha256(archive_bytes).hexdigest(), - organizations_count=7, - ) - - self.client.force_authenticate(self.admin) - response = self.client.post( - self.url, - {"actual_date": today.isoformat()}, - format="json", - ) - - self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.content, archive_bytes) - self.assertEqual(response["Content-Type"], "application/zip") - self.assertIn( - 'attachment; filename="backup.zip"', response["Content-Disposition"] - ) - self.assertEqual(response["X-Backup-Organizations"], "7") - self.assertEqual( - response["X-Backup-SHA256"], - hashlib.sha256(archive_bytes).hexdigest(), - ) - - self.assertFalse(archive_path.exists()) - self.assertFalse(BackupExportJob.objects.filter(id=job.id).exists()) - enqueue_mock.assert_not_called() - - @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") - def test_export_restarts_when_success_job_has_no_file(self, enqueue_mock): - today = timezone.localdate() - with TemporaryDirectory() as tmp_dir: - missing_archive_path = Path(tmp_dir) / "non-existent-backup.zip" - BackupExportJob.objects.create( - actual_date=today, - status=BackupExportJob.Status.SUCCESS, - task_id="task-old", - archive_path=str(missing_archive_path), - archive_filename="non-existent-backup.zip", - ) - - self.client.force_authenticate(self.admin) - with self.captureOnCommitCallbacks(execute=True): - response = self.client.post( - self.url, - {"actual_date": today.isoformat()}, - format="json", - ) - - self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - self.assertEqual(response.data["status"], "started") - restarted_job = BackupExportJob.objects.get(actual_date=today) - self.assertEqual(response.data["task_id"], restarted_job.task_id) - enqueue_mock.assert_called_once_with( - job_id=restarted_job.id, - task_id=restarted_job.task_id, - ) - - @patch("apps.backups.services.BackupExportJob.objects.create") - def test_check_or_start_job_handles_integrity_race(self, create_mock): - today = timezone.localdate() - concurrent_job = BackupExportJob( - actual_date=today, - status=BackupExportJob.Status.STARTED, - task_id="task-race-running", + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.content, b"zip-bytes") + self.assertEqual(response["Content-Type"], "application/zip") + self.assertIn( + 'attachment; filename="state_corp_exchange.zip"', + response["Content-Disposition"], ) - concurrent_job.save() - create_mock.side_effect = IntegrityError("duplicate key value") + self.assertEqual(response["Content-Length"], "9") + self.assertEqual(response["X-State-Corp-Package-Id"], "package-1") + self.assertEqual(response["X-State-Corp-Bin-File"], "state_corp_exchange.bin") + self.assertEqual(response["X-State-Corp-Organizations"], "2") + self.assertEqual(response["X-Backup-Organizations"], "2") + self.assertEqual(response["X-Backup-Actual-Date"], "2026-05-12") + build_package_mock.assert_called_once_with(actual_date=date(2026, 5, 12)) - with patch.object( - BackupExportJobService, - "_get_job_for_update", - side_effect=[None, concurrent_job], - ): - result = BackupExportJobService.check_or_start_job( - actual_date=today, - requested_by_id=self.admin.id, - ) - - self.assertEqual(result.action, "wait") - self.assertEqual(result.task_id, "task-race-running") - - @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") - def test_check_or_start_job_enqueues_on_commit(self, enqueue_mock): - today = timezone.localdate() - - with self.captureOnCommitCallbacks(execute=False) as callbacks: - result = BackupExportJobService.check_or_start_job( - actual_date=today, - requested_by_id=self.admin.id, - ) - - self.assertEqual(result.action, "started") - self.assertEqual(len(callbacks), 1) - enqueue_mock.assert_not_called() - - job = BackupExportJob.objects.get(actual_date=today) - self.assertEqual(job.task_id, result.task_id) - - callbacks[0]() - enqueue_mock.assert_called_once_with(job_id=job.id, task_id=job.task_id) - - @patch("apps.backups.views.BackupExportJobService.check_or_start_job") - def test_export_returns_400_when_job_start_fails(self, check_or_start_mock): + @patch("apps.backups.views.StateCorpExchangeService.build_package") + def test_export_returns_400_when_package_build_fails(self, build_package_mock): self.client.force_authenticate(self.admin) - check_or_start_mock.side_effect = BackupExportError("broken") + build_package_mock.side_effect = StateCorpExchangeError("broken") response = self.client.post(self.url, {}, format="json") @@ -206,32 +80,3 @@ class BackupExportViewTest(APITestCase): self.assertEqual( response.data["errors"][0]["details"]["fields"]["backup"], ["broken"] ) - - @patch("apps.backups.views.BackupExportJobService.consume_ready_archive") - @patch("apps.backups.views.BackupExportJobService.check_or_start_job") - def test_export_returns_400_when_archive_consumption_fails( - self, - check_or_start_mock, - consume_mock, - ): - today = timezone.localdate() - self.client.force_authenticate(self.admin) - check_or_start_mock.return_value = BackupRequestResult( - action="download", - message="ready", - actual_date=today, - task_id="task-ready", - ) - consume_mock.side_effect = BackupExportError("missing archive") - - response = self.client.post( - self.url, - {"actual_date": today.isoformat()}, - format="json", - ) - - self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) - self.assertEqual( - response.data["errors"][0]["details"]["fields"]["backup"], - ["missing archive"], - ) diff --git a/tests/apps/exchange/test_state_corp_services.py b/tests/apps/exchange/test_state_corp_services.py index 2902fce..d9b8c6c 100644 --- a/tests/apps/exchange/test_state_corp_services.py +++ b/tests/apps/exchange/test_state_corp_services.py @@ -12,6 +12,8 @@ from unittest.mock import Mock, patch from zipfile import ZipFile from apps.exchange.state_corp_services import StateCorpExchangeService +from apps.parsers.models import GenericParserRecord, ParserLoadLog +from apps.registers.models import Register from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.test import TestCase, override_settings @@ -20,7 +22,11 @@ from tests.apps.parsers.factories import ( InspectionRecordFactory, ProcurementRecordFactory, ) -from tests.apps.registers.factories import OrganizationFactory +from tests.apps.registers.factories import ( + OrganizationFactory, + RegisterFactory, + RegistryMembershipPeriodFactory, +) def _b64url_decode(value: str) -> bytes: @@ -28,6 +34,25 @@ def _b64url_decode(value: str) -> bytes: return base64.urlsafe_b64decode(normalized.encode("ascii")) +def _decode_package_payload(package): + with ZipFile(BytesIO(package.archive_bytes)) as archive: + bin_names = [name for name in archive.namelist() if name.endswith(".bin")] + assert len(bin_names) == 1 + bin_bytes = archive.read(bin_names[0]) + + header_size = struct.unpack(">I", bin_bytes[5:9])[0] + header_end = 9 + header_size + header = json.loads(bin_bytes[9:header_end].decode("utf-8")) + encrypted_payload = bin_bytes[header_end:] + raw_key = hashlib.sha256(TEST_STATE_CORP_TOKEN.encode("utf-8")).digest() + compressed_payload = AESGCM(raw_key).decrypt( + _b64url_decode(header["nonce"]), + encrypted_payload, + _b64url_decode(header["aad"]), + ) + return json.loads(zlib.decompress(compressed_payload).decode("utf-8")) + + TEST_STATE_CORP_TOKEN = "state-corp-test-exchange-token" # noqa: S105 @@ -39,6 +64,7 @@ class StateCorpExchangeServiceTest(TestCase): """Verify package compatibility with state-corp receiver contract.""" def test_build_package_contains_expected_payload(self): + registry = Register.objects.get(name="Реестр госкорпорации Росатом") organization = OrganizationFactory.create( mn_inn=7707083893, mn_ogrn=1027700132195, @@ -46,6 +72,12 @@ class StateCorpExchangeServiceTest(TestCase): in_kpp=770701001, mn_okpo="12345678", ) + RegistryMembershipPeriodFactory.create( + registry=registry, + organization=organization, + started_at="2026-01-01", + ended_at=None, + ) IndustrialProductRecordFactory.create( inn=str(organization.mn_inn), ogrn=str(organization.mn_ogrn), @@ -82,29 +114,100 @@ class StateCorpExchangeServiceTest(TestCase): max_price_amount="4500000.75", registry_organization=organization, ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.ARBITRATION, + load_batch=1, + external_id="case-001", + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + title="А40-1/2026", + record_date="2026-03-25", + status="in_progress", + payload={ + "case_number": "А40-1/2026", + "court_name": "АС города Москвы", + "target": {"role": "ответчик"}, + }, + registry_organization=organization, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY, + load_batch=1, + external_id="fedresurs-001", + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + title="Сообщение о намерении", + record_date="2026-03-26", + status="published", + url="https://fedresurs.ru/message/001", + payload={ + "type": "Сообщение о намерении", + "date": "2026-03-26", + "case_number": "А40-555/2026", + }, + registry_organization=organization, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.FAS_GOZ, + load_batch=1, + external_id="fas-goz-001", + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + title="Уклонение от заключения контракта", + record_date="2026-02-20", + status="active", + url="https://fas.gov.ru/register/001", + payload={ + "Номер реестровой записи": "ГОЗ-001", + "Полное наименование лица": organization.pn_name, + "Дата вступления постановления": "2026-02-20", + }, + registry_organization=organization, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.FSTEC, + load_batch=1, + external_id="fstec-001", + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + title="Реестр лицензий ФСТЭК", + record_date="2026-01-10", + status="present", + payload={ + "registry_name": "Реестр лицензий ФСТЭК", + "entry_number": "77-001234", + "issued_at": "2026-01-10", + "expires_at": "2027-01-10", + }, + registry_organization=organization, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.TRUDVSEM, + load_batch=1, + external_id="trudvsem-001", + inn=str(organization.mn_inn), + ogrn=str(organization.mn_ogrn), + title="Инженер-испытатель", + record_date="2026-04-01", + amount="175000.00", + status="open", + url="https://trudvsem.ru/vacancy/001", + payload={"vacancy_source": "trudvsem"}, + registry_organization=organization, + ) - package = StateCorpExchangeService.build_package() + package = StateCorpExchangeService.build_package(actual_date="2026-03-15") self.assertEqual(package.payload_counts["organizations"], 1) self.assertEqual(package.payload_counts["industrial_products"], 1) self.assertEqual(package.payload_counts["prosecutor_checks"], 1) self.assertEqual(package.payload_counts["public_procurements"], 1) + self.assertEqual(package.payload_counts["arbitration_cases"], 1) + self.assertEqual(package.payload_counts["bankruptcy_procedures"], 1) + self.assertEqual(package.payload_counts["defense_unreliable_suppliers"], 1) + self.assertEqual(package.payload_counts["information_security_registries"], 1) + self.assertEqual(package.payload_counts["labor_vacancies"], 1) - with ZipFile(BytesIO(package.archive_bytes)) as archive: - bin_names = [name for name in archive.namelist() if name.endswith(".bin")] - self.assertEqual(len(bin_names), 1) - bin_bytes = archive.read(bin_names[0]) - - header_size = struct.unpack(">I", bin_bytes[5:9])[0] - header_end = 9 + header_size - header = json.loads(bin_bytes[9:header_end].decode("utf-8")) - encrypted_payload = bin_bytes[header_end:] - raw_key = hashlib.sha256(TEST_STATE_CORP_TOKEN.encode("utf-8")).digest() - compressed_payload = AESGCM(raw_key).decrypt( - _b64url_decode(header["nonce"]), - encrypted_payload, - _b64url_decode(header["aad"]), - ) - payload = json.loads(zlib.decompress(compressed_payload).decode("utf-8")) + payload = _decode_package_payload(package) self.assertEqual(payload["format"], StateCorpExchangeService.PAYLOAD_FORMAT) self.assertEqual(payload["manifest"]["source_system"], "mostovik") @@ -121,7 +224,111 @@ class StateCorpExchangeServiceTest(TestCase): payload["data"]["public_procurements"][0]["purchase_number"], "purchase-001", ) - self.assertEqual(payload["data"]["arbitration_cases"], []) + self.assertEqual( + payload["data"]["arbitration_cases"][0]["case_number"], + "А40-1/2026", + ) + self.assertEqual( + payload["data"]["bankruptcy_procedures"][0]["case_number"], + "А40-555/2026", + ) + self.assertEqual( + payload["data"]["defense_unreliable_suppliers"][0]["registry_number"], + "ГОЗ-001", + ) + self.assertEqual( + payload["data"]["information_security_registries"][0]["entry_number"], + "77-001234", + ) + self.assertEqual( + payload["data"]["labor_vacancies"][0]["title"], + "Инженер-испытатель", + ) + + def test_build_package_exports_only_active_rosatom_roscosmos_registry_members(self): + target_registry = Register.objects.get(name="Реестр госкорпорации Роскосмос") + non_target_registry = RegisterFactory.create() + + target = OrganizationFactory.create( + mn_inn=7707000001, + mn_ogrn=1027700000001, + pn_name="АО Целевая", + ) + non_target = OrganizationFactory.create( + mn_inn=7707000002, + mn_ogrn=1027700000002, + pn_name="АО Не экспортируется", + ) + inactive_target = OrganizationFactory.create( + mn_inn=7707000003, + mn_ogrn=1027700000003, + pn_name="АО Бывшая", + ) + RegistryMembershipPeriodFactory.create( + registry=target_registry, + organization=target, + started_at="2026-01-01", + ended_at=None, + ) + RegistryMembershipPeriodFactory.create( + registry=non_target_registry, + organization=non_target, + started_at="2026-01-01", + ended_at=None, + ) + RegistryMembershipPeriodFactory.create( + registry=target_registry, + organization=inactive_target, + started_at="2026-01-01", + ended_at="2026-03-01", + ) + IndustrialProductRecordFactory.create( + inn=str(target.mn_inn), + registry_number="target-product", + registry_organization=target, + ) + IndustrialProductRecordFactory.create( + inn=str(non_target.mn_inn), + registry_number="non-target-product", + registry_organization=non_target, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.TRUDVSEM, + load_batch=1, + external_id="target-vacancy", + inn=str(target.mn_inn), + title="Целевая вакансия", + record_date="2026-03-10", + registry_organization=target, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.TRUDVSEM, + load_batch=1, + external_id="non-target-vacancy", + inn=str(non_target.mn_inn), + title="Лишняя вакансия", + record_date="2026-03-10", + registry_organization=non_target, + ) + + package = StateCorpExchangeService.build_package(actual_date="2026-03-15") + payload = _decode_package_payload(package) + + self.assertEqual( + [item["inn"] for item in payload["data"]["organizations"]], + [str(target.mn_inn)], + ) + self.assertEqual( + [ + item["registry_number"] + for item in payload["data"]["industrial_products"] + ], + ["target-product"], + ) + self.assertEqual( + [item["external_id"] for item in payload["data"]["labor_vacancies"]], + ["target-vacancy"], + ) @patch("apps.exchange.state_corp_services.requests.post") def test_send_package_posts_multipart_archive(self, post_mock): diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index 80c2ebc..ba41cec 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch from apps.core.models import BackgroundJob, JobStatus from apps.parsers.models import GenericParserRecord, ParserLoadLog from apps.parsers.source_cards import ( + SOURCE_CARD_DEFINITIONS, SourceCardDefinition, SourceCardService, SourceItemDefinition, @@ -18,6 +19,36 @@ from rest_framework.exceptions import ValidationError class SourceCardServiceUnitTest(SimpleTestCase): + def test_list_cards_exposes_all_frontend_category_slugs_in_menu_order(self): + self.assertEqual( + [card.slug for card in SOURCE_CARD_DEFINITIONS], + [ + "financial-indicators", + "public-procurements", + "manufacturers-and-products", + "planned-inspections", + "bankruptcy-procedures", + "defense-unreliable-suppliers", + "arbitration-cases", + "information-security-registries", + "labor-vacancies", + ], + ) + self.assertEqual( + [card.title for card in SOURCE_CARD_DEFINITIONS], + [ + "Финансово-экономические показатели", + "Государственные закупки по 44-ФЗ и 223-ФЗ", + "Производители и продукция России", + "Плановые проверки Генпрокуратуры России", + "Сведения о процедурах банкротства", + "Недобросовестные поставщики ГОЗ", + "Арбитражные дела", + "Реестры по информационной безопасности", + "Вакансии Работа России", + ], + ) + def test_get_definition_raises_for_unknown_slug(self): with self.assertRaises(Http404): SourceCardService.get_definition("missing-card") @@ -156,6 +187,40 @@ class SourceCardServiceUnitTest(SimpleTestCase): }, ) + @patch( + "apps.parsers.source_cards.SourceCardService._enqueue_task", + side_effect=[ + { + "task_id": "task-unfair", + "task_name": "apps.parsers.tasks.parse_unfair_suppliers", + }, + { + "task_id": "task-goz", + "task_name": "apps.parsers.tasks.parse_fas_goz_evasion", + }, + ], + ) + def test_refresh_card_for_defense_unreliable_suppliers_enqueues_sources( + self, enqueue_mock + ): + result = SourceCardService.refresh_card( + slug="defense-unreliable-suppliers", + requested_by_id=10, + ) + + self.assertEqual(result["source_card"], "defense-unreliable-suppliers") + self.assertEqual( + [item["task_id"] for item in result["tasks"]], + ["task-unfair", "task-goz"], + ) + self.assertEqual( + [call.kwargs["meta"]["source"] for call in enqueue_mock.call_args_list], + [ + ParserLoadLog.Source.UNFAIR_SUPPLIERS, + ParserLoadLog.Source.FAS_GOZ, + ], + ) + def test_launch_refresh_raises_for_unsupported_card(self): definition = SourceCardDefinition( slug="custom-source", @@ -297,6 +362,42 @@ class SourceCardServiceUnitTest(SimpleTestCase): @override_settings(PARSER_STALE_LOAD_MAX_AGE_MINUTES=90) class SourceCardServiceDatabaseTest(TestCase): + def test_defense_unreliable_suppliers_counts_unique_generic_organizations(self): + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.UNFAIR_SUPPLIERS, + load_batch=1, + external_id="unfair-1", + inn="7701234567", + title="Недобросовестный поставщик", + payload={"number": "unfair-1"}, + ) + GenericParserRecord.objects.create( + source=ParserLoadLog.Source.FAS_GOZ, + load_batch=1, + external_id="goz-1", + inn="7701234567", + title="Уклонение от ГОЗ", + payload={"number": "goz-1"}, + ) + ParserLoadLog.objects.create( + source=ParserLoadLog.Source.UNFAIR_SUPPLIERS, + batch_id=1, + records_count=1, + status=ParserLoadLog.Status.SUCCESS, + ) + ParserLoadLog.objects.create( + source=ParserLoadLog.Source.FAS_GOZ, + batch_id=1, + records_count=1, + status=ParserLoadLog.Status.SUCCESS, + ) + + card = SourceCardService.get_card("defense-unreliable-suppliers") + + self.assertEqual(card["status"], "success") + self.assertEqual(card["records_count"], 2) + self.assertEqual(card["organizations_count"], 1) + def test_public_procurements_counts_generic_eis_sources(self): GenericParserRecord.objects.create( source=ParserLoadLog.Source.PROCUREMENTS_44FZ, diff --git a/tests/apps/parsers/test_tasks.py b/tests/apps/parsers/test_tasks.py index 21a424c..d58a6b6 100644 --- a/tests/apps/parsers/test_tasks.py +++ b/tests/apps/parsers/test_tasks.py @@ -249,7 +249,9 @@ class GenericSourceFetchTestCase(TestCase): ) RegistryMembershipPeriodFactory(organization=organization, ended_at=None) RegistryMembershipPeriodFactory( - organization=inactive, ended_at=date(2026, 5, 7) + organization=inactive, + started_at=date(2026, 5, 1), + ended_at=date(2026, 5, 7), ) captured_inns = [] @@ -317,7 +319,9 @@ class GenericSourceFetchTestCase(TestCase): ) RegistryMembershipPeriodFactory(organization=organization, ended_at=None) RegistryMembershipPeriodFactory( - organization=inactive, ended_at=date(2026, 5, 7) + organization=inactive, + started_at=date(2026, 5, 1), + ended_at=date(2026, 5, 7), ) class _CheckoClient: @@ -448,7 +452,9 @@ class GenericSourceFetchTestCase(TestCase): ) RegistryMembershipPeriodFactory(organization=organization, ended_at=None) RegistryMembershipPeriodFactory( - organization=inactive, ended_at=date(2026, 5, 7) + organization=inactive, + started_at=date(2026, 5, 1), + ended_at=date(2026, 5, 7), ) class _CheckoClient: @@ -523,7 +529,9 @@ class GenericSourceFetchTestCase(TestCase): ) RegistryMembershipPeriodFactory(organization=organization, ended_at=None) RegistryMembershipPeriodFactory( - organization=inactive, ended_at=date(2026, 5, 7) + organization=inactive, + started_at=date(2026, 5, 1), + ended_at=date(2026, 5, 7), ) class _CheckoClient: @@ -2021,7 +2029,9 @@ class ParseVacanciesTaskTestCase(TestCase): RegistryMembershipPeriodFactory(organization=active_first, ended_at=None) RegistryMembershipPeriodFactory(organization=active_second, ended_at=None) RegistryMembershipPeriodFactory( - organization=inactive, ended_at=date(2026, 5, 7) + organization=inactive, + started_at=date(2026, 5, 1), + ended_at=date(2026, 5, 7), ) RegistryMembershipPeriodFactory(organization=active_first, ended_at=None) captured_fetches = [] diff --git a/tests/test_api_inventory_e2e.py b/tests/test_api_inventory_e2e.py index 98ff8e4..495749f 100644 --- a/tests/test_api_inventory_e2e.py +++ b/tests/test_api_inventory_e2e.py @@ -9,7 +9,6 @@ from tempfile import TemporaryDirectory from types import SimpleNamespace from unittest.mock import patch -from apps.backups.models import BackupExportJob from apps.core.models import BackgroundJob from apps.exchange.models import ExchangeConnection from apps.parsers.models import ( @@ -708,19 +707,29 @@ class BackupsApiInventoryE2ETest(AuthenticatedApiMixin, APITestCase): def setUp(self): self.admin = UserFactory.create_superuser() - @patch("apps.backups.services.BackupExportJobService._enqueue_backup_task") - def test_backup_export_endpoint(self, enqueue_mock): + @patch("apps.backups.views.StateCorpExchangeService.build_package") + def test_backup_export_endpoint(self, build_package_mock): self.authenticate(self.admin) today = timezone.localdate() + build_package_mock.return_value = SimpleNamespace( + package_id="package-1", + archive_name="state_corp_exchange.zip", + bin_name="state_corp_exchange.bin", + archive_bytes=b"zip-bytes", + payload_counts={"organizations": 1}, + produced_at="2026-05-12T10:00:00+00:00", + ) - with self.captureOnCommitCallbacks(execute=True): - response = self.client.post( - reverse("api_v1:backups:export"), - {"actual_date": today.isoformat()}, - format="json", - ) + response = self.client.post( + reverse("api_v1:backups:export"), + {"actual_date": today.isoformat()}, + format="json", + ) - self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED) - self.assertEqual(response.data["status"], "started") - self.assertTrue(BackupExportJob.objects.filter(actual_date=today).exists()) - enqueue_mock.assert_called_once() + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.content, b"zip-bytes") + self.assertIn( + 'attachment; filename="state_corp_exchange.zip"', + response["Content-Disposition"], + ) + build_package_mock.assert_called_once_with(actual_date=today)