feat: export state corp package from backup endpoint
This commit is contained in:
@@ -1,8 +1,11 @@
|
||||
"""API views экспорта защищённых backup архивов."""
|
||||
|
||||
from apps.backups.serializers import BackupExportRequestSerializer
|
||||
from apps.backups.services import BackupExportError, BackupExportJobService
|
||||
from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag
|
||||
from apps.exchange.state_corp_services import (
|
||||
StateCorpExchangeError,
|
||||
StateCorpExchangeService,
|
||||
)
|
||||
from django.http import HttpResponse
|
||||
from django.utils import timezone
|
||||
from drf_yasg import openapi
|
||||
@@ -10,14 +13,13 @@ from drf_yasg.utils import swagger_auto_schema
|
||||
from rest_framework import status
|
||||
from rest_framework.exceptions import ValidationError
|
||||
from rest_framework.permissions import IsAdminUser
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.views import APIView
|
||||
|
||||
BACKUPS_TAG = swagger_tag("Резервные копии", "backups")
|
||||
|
||||
|
||||
class BackupExportView(APIView):
|
||||
"""Асинхронный экспорт защищённого backup архива."""
|
||||
"""Синхронный экспорт защищённого ZIP-пакета для state-corp."""
|
||||
|
||||
permission_classes = [IsAdminUser]
|
||||
|
||||
@@ -25,16 +27,12 @@ class BackupExportView(APIView):
|
||||
tags=[BACKUPS_TAG],
|
||||
operation_summary="Сформировать backup архив",
|
||||
operation_description=(
|
||||
"Асинхронное формирование архива `.zip` на дату актуальности.\n"
|
||||
"Логика endpoint:\n"
|
||||
"- если задача на дату уже выполняется: вернуть 'подождите'\n"
|
||||
"- если задачи нет: запустить формирование\n"
|
||||
"- если задача завершена успешно: отдать архив и удалить его\n\n"
|
||||
"Синхронное формирование архива `.zip` на дату актуальности.\n"
|
||||
"Внутри архива:\n"
|
||||
"- бинарный зашифрованный backup `.bin`\n"
|
||||
"- бинарный зашифрованный пакет `.bin`\n"
|
||||
"- контрольная сумма `.sha256`\n\n"
|
||||
"Экспортирует только актуальные организации из реестров на указанную дату "
|
||||
"и все связанные записи из parser-таблиц."
|
||||
"Экспортирует организации из реестров Росатом и Роскосмос на указанную "
|
||||
"дату и связанные записи из parser-таблиц в контракте импорта state-corp."
|
||||
),
|
||||
request_body=BackupExportRequestSerializer,
|
||||
responses={
|
||||
@@ -42,21 +40,6 @@ class BackupExportView(APIView):
|
||||
description="Готовый backup-архив (application/zip)",
|
||||
schema=openapi.Schema(type=openapi.TYPE_FILE),
|
||||
),
|
||||
202: openapi.Response(
|
||||
description="Задача запущена или выполняется",
|
||||
schema=openapi.Schema(
|
||||
type=openapi.TYPE_OBJECT,
|
||||
properties={
|
||||
"status": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
"message": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
"actual_date": openapi.Schema(
|
||||
type=openapi.TYPE_STRING,
|
||||
format=openapi.FORMAT_DATE,
|
||||
),
|
||||
"task_id": openapi.Schema(type=openapi.TYPE_STRING),
|
||||
},
|
||||
),
|
||||
),
|
||||
400: CommonResponses.BAD_REQUEST,
|
||||
**ErrorResponses.ADMIN,
|
||||
},
|
||||
@@ -69,40 +52,23 @@ class BackupExportView(APIView):
|
||||
)
|
||||
|
||||
try:
|
||||
result = BackupExportJobService.check_or_start_job(
|
||||
actual_date=actual_date,
|
||||
requested_by_id=request.user.id
|
||||
if request.user.is_authenticated
|
||||
else None,
|
||||
)
|
||||
except BackupExportError as exc:
|
||||
package = StateCorpExchangeService.build_package(actual_date=actual_date)
|
||||
except StateCorpExchangeError as exc:
|
||||
raise ValidationError({"backup": str(exc)}) from exc
|
||||
|
||||
if result.action in {"started", "wait"}:
|
||||
return Response(
|
||||
{
|
||||
"status": result.action,
|
||||
"message": result.message,
|
||||
"actual_date": result.actual_date.isoformat(),
|
||||
"task_id": result.task_id,
|
||||
},
|
||||
status=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
|
||||
try:
|
||||
artifact = BackupExportJobService.consume_ready_archive(
|
||||
actual_date=result.actual_date
|
||||
)
|
||||
except BackupExportError as exc:
|
||||
raise ValidationError({"backup": str(exc)}) from exc
|
||||
|
||||
response = HttpResponse(artifact.archive_bytes, content_type="application/zip")
|
||||
response = HttpResponse(package.archive_bytes, content_type="application/zip")
|
||||
response.status_code = status.HTTP_200_OK
|
||||
response[
|
||||
"Content-Disposition"
|
||||
] = f'attachment; filename="{artifact.archive_filename}"'
|
||||
response["X-Backup-SHA256"] = artifact.checksum_sha256
|
||||
response["X-Backup-Checksum-File"] = artifact.checksum_filename
|
||||
response["X-Backup-Organizations"] = str(artifact.organizations_count)
|
||||
response["X-Backup-Actual-Date"] = artifact.actual_date.isoformat()
|
||||
] = f'attachment; filename="{package.archive_name}"'
|
||||
response["Content-Length"] = str(len(package.archive_bytes))
|
||||
response["X-State-Corp-Package-Id"] = package.package_id
|
||||
response["X-State-Corp-Bin-File"] = package.bin_name
|
||||
response["X-State-Corp-Organizations"] = str(
|
||||
package.payload_counts.get("organizations", 0)
|
||||
)
|
||||
response["X-Backup-Organizations"] = str(
|
||||
package.payload_counts.get("organizations", 0)
|
||||
)
|
||||
response["X-Backup-Actual-Date"] = actual_date.isoformat()
|
||||
return response
|
||||
|
||||
@@ -19,14 +19,17 @@ from zipfile import ZIP_DEFLATED, ZipFile
|
||||
|
||||
import requests
|
||||
from apps.parsers.models import (
|
||||
GenericParserRecord,
|
||||
IndustrialProductRecord,
|
||||
InspectionRecord,
|
||||
ParserLoadLog,
|
||||
ProcurementRecord,
|
||||
)
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
from django.conf import settings
|
||||
from django.db.models import Q
|
||||
from django.utils import timezone
|
||||
from registers.models import Organization
|
||||
from registers.models import Organization, RegistryMembershipPeriod
|
||||
|
||||
|
||||
class StateCorpExchangeError(ValueError):
|
||||
@@ -52,6 +55,14 @@ class StateCorpExchangeService:
|
||||
AAD = b"state-corp-exchange-v1"
|
||||
PAYLOAD_FORMAT = "state-corp-exchange-payload"
|
||||
BIN_FORMAT = "state-corp-exchange-bin"
|
||||
ROSATOM_ROSCOSMOS_REGISTRY_NAMES = (
|
||||
"Реестр госкорпорации Роскосмос",
|
||||
"Реестр госкорпорации Роскосмос ГОЗ",
|
||||
"Реестр госкорпорации Роскосмос ОПК",
|
||||
"Реестр госкорпорации Росатом",
|
||||
"Реестр госкорпорации Росатом ГОЗ",
|
||||
"Реестр госкорпорации Росатом ОПК",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build_package(
|
||||
@@ -60,18 +71,32 @@ class StateCorpExchangeService:
|
||||
package_id: str | None = None,
|
||||
source_system: str = "mostovik",
|
||||
organization_inns: list[str] | None = None,
|
||||
actual_date: date | str | None = None,
|
||||
) -> StateCorpExchangePackage:
|
||||
"""Build encrypted archive for state-corp import."""
|
||||
produced_at = timezone.now()
|
||||
snapshot_date = cls._coerce_actual_date(actual_date)
|
||||
normalized_inns = cls._normalize_inn_list(organization_inns)
|
||||
organizations = cls._get_organizations(normalized_inns)
|
||||
organizations = (
|
||||
cls._get_organizations(normalized_inns)
|
||||
if normalized_inns
|
||||
else cls._get_rosatom_roscosmos_organizations(snapshot_date)
|
||||
)
|
||||
allowed_inns = {str(item.mn_inn) for item in organizations}
|
||||
data = {
|
||||
"organizations": cls._serialize_organizations(organizations),
|
||||
"industrial_products": cls._serialize_industrial_products(allowed_inns),
|
||||
"prosecutor_checks": cls._serialize_prosecutor_checks(allowed_inns),
|
||||
"public_procurements": cls._serialize_public_procurements(allowed_inns),
|
||||
"arbitration_cases": [],
|
||||
"arbitration_cases": cls._serialize_arbitration_cases(allowed_inns),
|
||||
"bankruptcy_procedures": cls._serialize_bankruptcy_procedures(allowed_inns),
|
||||
"defense_unreliable_suppliers": (
|
||||
cls._serialize_defense_unreliable_suppliers(allowed_inns)
|
||||
),
|
||||
"information_security_registries": (
|
||||
cls._serialize_information_security_registries(allowed_inns)
|
||||
),
|
||||
"labor_vacancies": cls._serialize_labor_vacancies(allowed_inns),
|
||||
}
|
||||
payload_counts = {key: len(value) for key, value in data.items()}
|
||||
package_id = package_id or cls._build_package_id()
|
||||
@@ -248,6 +273,23 @@ class StateCorpExchangeService:
|
||||
)
|
||||
return list(queryset)
|
||||
|
||||
@classmethod
|
||||
def _get_rosatom_roscosmos_organizations(
|
||||
cls,
|
||||
actual_date: date,
|
||||
) -> list[Organization]:
|
||||
organization_ids = (
|
||||
RegistryMembershipPeriod.objects.filter(
|
||||
registry__name__in=cls.ROSATOM_ROSCOSMOS_REGISTRY_NAMES,
|
||||
started_at__lte=actual_date,
|
||||
)
|
||||
.filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date))
|
||||
.order_by("organization_id")
|
||||
.values_list("organization_id", flat=True)
|
||||
.distinct()
|
||||
)
|
||||
return list(Organization.objects.filter(id__in=organization_ids).order_by("id"))
|
||||
|
||||
@classmethod
|
||||
def _serialize_organizations(
|
||||
cls,
|
||||
@@ -350,6 +392,249 @@ class StateCorpExchangeService:
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _serialize_arbitration_cases(
|
||||
cls,
|
||||
allowed_inns: set[str],
|
||||
) -> list[dict[str, str]]:
|
||||
items: list[dict[str, str]] = []
|
||||
queryset = GenericParserRecord.objects.filter(
|
||||
source=ParserLoadLog.Source.ARBITRATION,
|
||||
inn__in=allowed_inns,
|
||||
).order_by("id")
|
||||
for record in queryset:
|
||||
payload = record.payload if isinstance(record.payload, dict) else {}
|
||||
target = payload.get("target")
|
||||
if not isinstance(target, dict):
|
||||
target = {}
|
||||
|
||||
decision_date = cls._coerce_date(
|
||||
None,
|
||||
payload.get("result_date")
|
||||
or payload.get("decision_date")
|
||||
or payload.get("filing_date")
|
||||
or record.record_date,
|
||||
)
|
||||
case_number = str(
|
||||
payload.get("case_number") or record.title or record.external_id or ""
|
||||
).strip()
|
||||
if not case_number or decision_date is None:
|
||||
continue
|
||||
|
||||
items.append(
|
||||
{
|
||||
"organization_inn": cls._digits(record.inn),
|
||||
"case_number": case_number,
|
||||
"court_name": str(payload.get("court_name") or "").strip(),
|
||||
"party_role": str(
|
||||
target.get("role") or payload.get("party_role") or ""
|
||||
).strip(),
|
||||
"status": str(payload.get("status") or record.status or "").strip(),
|
||||
"decision_date": decision_date.isoformat(),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _serialize_bankruptcy_procedures(
|
||||
cls,
|
||||
allowed_inns: set[str],
|
||||
) -> list[dict[str, str | None]]:
|
||||
items: list[dict[str, str | None]] = []
|
||||
for record in cls._generic_records(
|
||||
allowed_inns,
|
||||
sources=[ParserLoadLog.Source.FEDRESURS_BANKRUPTCY],
|
||||
):
|
||||
payload = cls._record_payload(record)
|
||||
message_date = cls._coerce_date(
|
||||
None,
|
||||
cls._payload_lookup(payload, ["message_date", "date", "Дата"])
|
||||
or record.record_date,
|
||||
)
|
||||
message_type = cls._payload_lookup(payload, ["message_type", "type"])
|
||||
items.append(
|
||||
{
|
||||
"organization_inn": cls._digits(record.inn),
|
||||
"external_id": record.external_id,
|
||||
"message_type": message_type or record.title or record.status,
|
||||
"message_date": message_date.isoformat() if message_date else None,
|
||||
"case_number": cls._payload_lookup(payload, ["case_number"]),
|
||||
"status": record.status,
|
||||
"source_url": record.url,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _serialize_defense_unreliable_suppliers(
|
||||
cls,
|
||||
allowed_inns: set[str],
|
||||
) -> list[dict[str, str | None]]:
|
||||
items: list[dict[str, str | None]] = []
|
||||
for record in cls._generic_records(
|
||||
allowed_inns,
|
||||
sources=[
|
||||
ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
ParserLoadLog.Source.FAS_GOZ,
|
||||
],
|
||||
):
|
||||
payload = cls._record_payload(record)
|
||||
included_at = cls._coerce_date(
|
||||
None,
|
||||
cls._payload_lookup(
|
||||
payload,
|
||||
[
|
||||
"included_at",
|
||||
"date",
|
||||
"Дата вступления постановления",
|
||||
"Включено",
|
||||
],
|
||||
)
|
||||
or record.record_date,
|
||||
)
|
||||
items.append(
|
||||
{
|
||||
"organization_inn": cls._digits(record.inn),
|
||||
"external_id": record.external_id,
|
||||
"registry_source": record.source,
|
||||
"registry_number": cls._payload_lookup(
|
||||
payload,
|
||||
[
|
||||
"registry_number",
|
||||
"number",
|
||||
"Номер реестровой записи",
|
||||
"Номер реестровой записи в ЕРУЗ",
|
||||
],
|
||||
),
|
||||
"supplier_name": cls._payload_lookup(
|
||||
payload,
|
||||
[
|
||||
"supplier_name",
|
||||
"organisation_name",
|
||||
"Полное наименование лица",
|
||||
"Фирменное наименование лица",
|
||||
"Наименование ФИО недобросовестного поставщика",
|
||||
],
|
||||
)
|
||||
or record.organisation_name,
|
||||
"reason": record.title,
|
||||
"included_at": included_at.isoformat() if included_at else None,
|
||||
"status": record.status,
|
||||
"source_url": record.url,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _serialize_information_security_registries(
|
||||
cls,
|
||||
allowed_inns: set[str],
|
||||
) -> list[dict[str, str | None]]:
|
||||
items: list[dict[str, str | None]] = []
|
||||
for record in cls._generic_records(
|
||||
allowed_inns,
|
||||
sources=[ParserLoadLog.Source.FSTEC],
|
||||
):
|
||||
payload = cls._record_payload(record)
|
||||
issued_at = cls._coerce_date(
|
||||
None,
|
||||
cls._payload_lookup(
|
||||
payload,
|
||||
["issued_at", "date", "Дата предоставления лицензии"],
|
||||
)
|
||||
or record.record_date,
|
||||
)
|
||||
expires_at = cls._coerce_date(
|
||||
None,
|
||||
cls._payload_lookup(
|
||||
payload,
|
||||
["expires_at", "Срок действия сертификата", "res_valid_till"],
|
||||
),
|
||||
)
|
||||
items.append(
|
||||
{
|
||||
"organization_inn": cls._digits(record.inn),
|
||||
"external_id": record.external_id,
|
||||
"registry_name": cls._payload_lookup(payload, ["registry_name"])
|
||||
or record.title
|
||||
or "Реестр ФСТЭК",
|
||||
"presence_status": "present",
|
||||
"entry_number": cls._payload_lookup(
|
||||
payload,
|
||||
[
|
||||
"entry_number",
|
||||
"registration_number",
|
||||
"Регистрационный номер лицензии",
|
||||
"№ сертификата",
|
||||
],
|
||||
)
|
||||
or record.external_id,
|
||||
"issued_at": issued_at.isoformat() if issued_at else None,
|
||||
"expires_at": expires_at.isoformat() if expires_at else None,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
@classmethod
|
||||
def _serialize_labor_vacancies(
|
||||
cls,
|
||||
allowed_inns: set[str],
|
||||
) -> list[dict[str, str | None]]:
|
||||
items: list[dict[str, str | None]] = []
|
||||
for record in cls._generic_records(
|
||||
allowed_inns,
|
||||
sources=[ParserLoadLog.Source.TRUDVSEM],
|
||||
):
|
||||
payload = cls._record_payload(record)
|
||||
published_at = cls._coerce_date(
|
||||
None,
|
||||
cls._payload_lookup(
|
||||
payload,
|
||||
["published_at", "creation-date", "date", "published_at"],
|
||||
)
|
||||
or record.record_date,
|
||||
)
|
||||
items.append(
|
||||
{
|
||||
"organization_inn": cls._digits(record.inn),
|
||||
"external_id": record.external_id,
|
||||
"vacancy_source": cls._payload_lookup(payload, ["vacancy_source"])
|
||||
or "trudvsem",
|
||||
"title": record.title,
|
||||
"status": record.status,
|
||||
"published_at": published_at.isoformat() if published_at else None,
|
||||
"salary_amount": cls._serialize_decimal(record.amount),
|
||||
"source_url": record.url,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def _record_payload(record: GenericParserRecord) -> dict[str, Any]:
|
||||
return record.payload if isinstance(record.payload, dict) else {}
|
||||
|
||||
@classmethod
|
||||
def _generic_records(
|
||||
cls,
|
||||
allowed_inns: set[str],
|
||||
*,
|
||||
sources: list[str],
|
||||
):
|
||||
if not allowed_inns:
|
||||
return GenericParserRecord.objects.none()
|
||||
return GenericParserRecord.objects.filter(
|
||||
source__in=sources,
|
||||
inn__in=allowed_inns,
|
||||
).order_by("id")
|
||||
|
||||
@staticmethod
|
||||
def _payload_lookup(payload: dict[str, Any], candidates: list[str]) -> str:
|
||||
for key in candidates:
|
||||
value = payload.get(key)
|
||||
if value not in (None, ""):
|
||||
return str(value).strip()
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _normalize_inn_list(organization_inns: list[str] | None) -> list[str]:
|
||||
normalized_items: list[str] = []
|
||||
@@ -359,6 +644,20 @@ class StateCorpExchangeService:
|
||||
normalized_items.append(normalized)
|
||||
return normalized_items
|
||||
|
||||
@staticmethod
|
||||
def _coerce_actual_date(value: date | str | None) -> date:
|
||||
if value is None:
|
||||
return timezone.localdate()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
raw_text = str(value).strip()
|
||||
try:
|
||||
return date.fromisoformat(raw_text)
|
||||
except ValueError as exc:
|
||||
raise StateCorpExchangeError(
|
||||
"actual_date должен быть датой в формате YYYY-MM-DD"
|
||||
) from exc
|
||||
|
||||
@staticmethod
|
||||
def _digits(value: str | int) -> str:
|
||||
return "".join(char for char in str(value).strip() if char.isdigit())
|
||||
|
||||
@@ -218,6 +218,94 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
),
|
||||
),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="bankruptcy-procedures",
|
||||
title="Сведения о процедурах банкротства",
|
||||
description="Сведения Федресурса о процедурах банкротства.",
|
||||
order=50,
|
||||
task_names=("apps.parsers.tasks.parse_fedresurs_bankruptcy",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="fedresurs_bankruptcy",
|
||||
title="Банкротства Федресурс",
|
||||
description="Сведения о процедурах банкротства из ЕФРСБ.",
|
||||
parser_source=ParserLoadLog.Source.FEDRESURS_BANKRUPTCY,
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=1),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="defense-unreliable-suppliers",
|
||||
title="Недобросовестные поставщики ГОЗ",
|
||||
description="Реестры недобросовестных поставщиков и уклонения от ГОЗ.",
|
||||
order=60,
|
||||
task_names=(
|
||||
"apps.parsers.tasks.parse_unfair_suppliers",
|
||||
"apps.parsers.tasks.parse_fas_goz_evasion",
|
||||
),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="unfair_suppliers",
|
||||
title="Недобросовестные поставщики",
|
||||
description="Реестр недобросовестных поставщиков ФАС и ЕИС.",
|
||||
parser_source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
),
|
||||
SourceItemDefinition(
|
||||
code="fas_goz",
|
||||
title="Уклонение от ГОЗ",
|
||||
description="Юрлица, привлеченные за отказ или уклонение от ГОЗ.",
|
||||
parser_source=ParserLoadLog.Source.FAS_GOZ,
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=1),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="arbitration-cases",
|
||||
title="Арбитражные дела",
|
||||
description="Арбитражные дела по организациям.",
|
||||
order=70,
|
||||
task_names=("apps.parsers.tasks.parse_arbitration_cases",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="arbitration",
|
||||
title="Арбитражные дела",
|
||||
description="Карточки арбитражных дел по ИНН/ОГРН организаций.",
|
||||
parser_source=ParserLoadLog.Source.ARBITRATION,
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=1),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="information-security-registries",
|
||||
title="Реестры по информационной безопасности",
|
||||
description="Реестры ФСТЭК по информационной безопасности.",
|
||||
order=80,
|
||||
task_names=("apps.parsers.tasks.parse_fstec_registers",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="fstec",
|
||||
title="Реестры ФСТЭК",
|
||||
description="Официальные реестры ФСТЭК по информационной безопасности.",
|
||||
parser_source=ParserLoadLog.Source.FSTEC,
|
||||
),
|
||||
),
|
||||
refresh_interval=timedelta(days=30),
|
||||
),
|
||||
SourceCardDefinition(
|
||||
slug="labor-vacancies",
|
||||
title="Вакансии Работа России",
|
||||
description="Вакансии работодателей, состоящих в активных реестрах.",
|
||||
order=90,
|
||||
task_names=("apps.parsers.tasks.parse_trudvsem_vacancies",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="trudvsem",
|
||||
title="Вакансии Работа России",
|
||||
description="Вакансии работодателей из Работа России, HH и SuperJob.",
|
||||
parser_source=ParserLoadLog.Source.TRUDVSEM,
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS}
|
||||
@@ -231,6 +319,12 @@ GENERIC_RECORD_SOURCES_BY_ITEM_CODE = {
|
||||
"procurements_44fz": ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
"procurements_223fz": ParserLoadLog.Source.PROCUREMENTS_223FZ,
|
||||
"contracts": ParserLoadLog.Source.CONTRACTS,
|
||||
"unfair_suppliers": ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
"fas_goz": ParserLoadLog.Source.FAS_GOZ,
|
||||
"arbitration": ParserLoadLog.Source.ARBITRATION,
|
||||
"fedresurs_bankruptcy": ParserLoadLog.Source.FEDRESURS_BANKRUPTCY,
|
||||
"fstec": ParserLoadLog.Source.FSTEC,
|
||||
"trudvsem": ParserLoadLog.Source.TRUDVSEM,
|
||||
}
|
||||
|
||||
|
||||
@@ -545,8 +639,81 @@ class SourceCardService:
|
||||
)
|
||||
return [task_info]
|
||||
|
||||
task_specs = cls._get_simple_refresh_task_specs(definition.slug)
|
||||
if task_specs:
|
||||
return [
|
||||
cls._enqueue_task(
|
||||
task=task,
|
||||
task_name=task_name,
|
||||
requested_by_id=requested_by_id,
|
||||
meta={
|
||||
"source_card": definition.slug,
|
||||
"source": parser_source,
|
||||
},
|
||||
kwargs={"requested_by_id": requested_by_id},
|
||||
)
|
||||
for task, task_name, parser_source in task_specs
|
||||
]
|
||||
|
||||
raise ValidationError({"detail": "Обновление для карточки не поддерживается."})
|
||||
|
||||
@staticmethod
|
||||
def _get_simple_refresh_task_specs(
|
||||
slug: str,
|
||||
) -> tuple[tuple[Any, str, str], ...]:
|
||||
from apps.parsers.tasks import (
|
||||
parse_arbitration_cases,
|
||||
parse_fas_goz_evasion,
|
||||
parse_fedresurs_bankruptcy,
|
||||
parse_fstec_registers,
|
||||
parse_trudvsem_vacancies,
|
||||
parse_unfair_suppliers,
|
||||
)
|
||||
|
||||
specs = {
|
||||
"bankruptcy-procedures": (
|
||||
(
|
||||
parse_fedresurs_bankruptcy,
|
||||
"apps.parsers.tasks.parse_fedresurs_bankruptcy",
|
||||
ParserLoadLog.Source.FEDRESURS_BANKRUPTCY,
|
||||
),
|
||||
),
|
||||
"defense-unreliable-suppliers": (
|
||||
(
|
||||
parse_unfair_suppliers,
|
||||
"apps.parsers.tasks.parse_unfair_suppliers",
|
||||
ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
),
|
||||
(
|
||||
parse_fas_goz_evasion,
|
||||
"apps.parsers.tasks.parse_fas_goz_evasion",
|
||||
ParserLoadLog.Source.FAS_GOZ,
|
||||
),
|
||||
),
|
||||
"arbitration-cases": (
|
||||
(
|
||||
parse_arbitration_cases,
|
||||
"apps.parsers.tasks.parse_arbitration_cases",
|
||||
ParserLoadLog.Source.ARBITRATION,
|
||||
),
|
||||
),
|
||||
"information-security-registries": (
|
||||
(
|
||||
parse_fstec_registers,
|
||||
"apps.parsers.tasks.parse_fstec_registers",
|
||||
ParserLoadLog.Source.FSTEC,
|
||||
),
|
||||
),
|
||||
"labor-vacancies": (
|
||||
(
|
||||
parse_trudvsem_vacancies,
|
||||
"apps.parsers.tasks.parse_trudvsem_vacancies",
|
||||
ParserLoadLog.Source.TRUDVSEM,
|
||||
),
|
||||
),
|
||||
}
|
||||
return specs.get(slug, ())
|
||||
|
||||
@classmethod
|
||||
def _enqueue_task(
|
||||
cls,
|
||||
@@ -717,6 +884,7 @@ class SourceCardService:
|
||||
source_items: list[dict[str, Any]],
|
||||
) -> int:
|
||||
if definition.slug == "public-procurements":
|
||||
generic_sources = cls._get_generic_sources_for_definition(definition)
|
||||
legacy_inns = (
|
||||
ProcurementRecord.objects.exclude(customer_inn="")
|
||||
.order_by()
|
||||
@@ -724,9 +892,7 @@ class SourceCardService:
|
||||
.distinct()
|
||||
)
|
||||
generic_inns = (
|
||||
GenericParserRecord.objects.filter(
|
||||
source__in=GENERIC_RECORD_SOURCES_BY_ITEM_CODE.values()
|
||||
)
|
||||
GenericParserRecord.objects.filter(source__in=generic_sources)
|
||||
.exclude(inn="")
|
||||
.order_by()
|
||||
.values_list("inn", flat=True)
|
||||
@@ -735,6 +901,15 @@ class SourceCardService:
|
||||
return legacy_inns.union(generic_inns).count()
|
||||
|
||||
if definition.slug != "manufacturers-and-products":
|
||||
generic_sources = cls._get_generic_sources_for_definition(definition)
|
||||
if generic_sources and len(generic_sources) == len(definition.source_items):
|
||||
return (
|
||||
GenericParserRecord.objects.filter(source__in=generic_sources)
|
||||
.exclude(inn="")
|
||||
.values("inn")
|
||||
.distinct()
|
||||
.count()
|
||||
)
|
||||
return sum(item["organizations_count"] for item in source_items)
|
||||
|
||||
industrial_inns = (
|
||||
@@ -757,6 +932,18 @@ class SourceCardService:
|
||||
)
|
||||
return industrial_inns.union(manufacturer_inns, product_inns).count()
|
||||
|
||||
@staticmethod
|
||||
def _get_generic_sources_for_definition(
|
||||
definition: SourceCardDefinition,
|
||||
) -> list[str]:
|
||||
return list(
|
||||
dict.fromkeys(
|
||||
GENERIC_RECORD_SOURCES_BY_ITEM_CODE[item.code]
|
||||
for item in definition.source_items
|
||||
if item.code in GENERIC_RECORD_SOURCES_BY_ITEM_CODE
|
||||
)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_latest_load(
|
||||
cls,
|
||||
|
||||
@@ -1847,8 +1847,8 @@
|
||||
<section id="backupPanel">
|
||||
<div class="section-head">
|
||||
<div>
|
||||
<h2>Выгрузка организаций реестра в .bin</h2>
|
||||
<p>Формирование защищенного архива по выбранному реестру и дате актуальности.</p>
|
||||
<h2>Выгрузка организаций Росатом/Роскосмос в ZIP</h2>
|
||||
<p>Синхронное формирование защищенного ZIP-архива для импорта в state-corp.</p>
|
||||
</div>
|
||||
</div>
|
||||
<form id="backupExportForm">
|
||||
@@ -1857,11 +1857,11 @@
|
||||
<label>Дата актуальности<input name="actual_date" id="backupActualDate" type="date"></label>
|
||||
</div>
|
||||
<div class="row">
|
||||
<button type="submit">Сформировать / скачать .bin</button>
|
||||
<button type="submit">Сформировать / скачать ZIP</button>
|
||||
<span class="muted">Запрос уйдет в <code>/api/v1/backups/export/</code></span>
|
||||
</div>
|
||||
</form>
|
||||
<div id="backupStatus" class="empty-state">Выберите реестр и дату для защищенной выгрузки.</div>
|
||||
<div id="backupStatus" class="empty-state">Выберите дату для защищенной выгрузки.</div>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
@@ -2914,7 +2914,7 @@
|
||||
["Registers", "Registries", "GET", "/api/v1/registers/registries/?page_size=1"],
|
||||
["Registers", "Organizations", "GET", "/api/v1/registers/organizations/?page_size=1"],
|
||||
["Registers", "Upload registry", "POST", "/api/v2/registers/opk/upload/"],
|
||||
["Backups", "Export .bin", "POST", "/api/v1/backups/export/"],
|
||||
["Backups", "Export state-corp ZIP", "POST", "/api/v1/backups/export/"],
|
||||
["Exchange", "Connections", "GET", "/api/v1/exchange/connections/"],
|
||||
["Exchange", "Create connection", "POST", "/api/v1/exchange/connections/"],
|
||||
["Exchange", "Test connection", "POST", "/api/v1/exchange/connections/test/"],
|
||||
@@ -4303,7 +4303,7 @@
|
||||
$("organizationRegistryFilter").innerHTML = `<option value="">Все реестры</option>`;
|
||||
renderOrganizationDataSourceOptions();
|
||||
$("registrySummary").innerHTML = `<div class="empty-state">Реестры недоступны: ${escapeHtml(errorMessage(error))}</div>`;
|
||||
$("backupStatus").textContent = `Выгрузка .bin недоступна: ${errorMessage(error)}`;
|
||||
$("backupStatus").textContent = `Выгрузка ZIP недоступна: ${errorMessage(error)}`;
|
||||
}
|
||||
|
||||
function setSelectOptions(selectId, tables) {
|
||||
@@ -4899,7 +4899,7 @@
|
||||
return;
|
||||
}
|
||||
const payload = result.data || result;
|
||||
$("backupStatus").textContent = `${payload.status || "queued"}: ${payload.message || ""} task_id=${payload.task_id || ""}`;
|
||||
$("backupStatus").textContent = `${payload.status || "done"}: ${payload.message || ""}`;
|
||||
await refreshDashboard();
|
||||
} catch (error) {
|
||||
$("backupStatus").textContent = JSON.stringify(error);
|
||||
|
||||
Reference in New Issue
Block a user