1049 lines
36 KiB
Python
1049 lines
36 KiB
Python
"""Сериализаторы для приложения парсеров."""
|
||
|
||
import ipaddress
|
||
import socket
|
||
from urllib.parse import urlsplit
|
||
|
||
from apps.parsers.clients.common.structured import (
|
||
MAX_FILE_SIZE_BYTES,
|
||
SUPPORTED_EXCEL_EXTENSIONS,
|
||
SUPPORTED_ZIP_EXTENSIONS,
|
||
)
|
||
from apps.parsers.fns_upload import FNS_XLSX_FILENAME_RE
|
||
from apps.parsers.models import (
|
||
FinancialReport,
|
||
FinancialReportLine,
|
||
GenericParserRecord,
|
||
IndustrialCertificateRecord,
|
||
IndustrialProductRecord,
|
||
InspectionRecord,
|
||
ManufacturerRecord,
|
||
ParserLoadLog,
|
||
ParsingSettings,
|
||
ProcurementRecord,
|
||
)
|
||
from apps.parsers.organization_enrichment import (
|
||
ORGANIZATION_ENRICHMENT_CONTEXT_KEY,
|
||
OrganizationFieldSet,
|
||
build_canonical_organization_enrichment,
|
||
enrich_organization_fields,
|
||
)
|
||
from rest_framework import serializers
|
||
|
||
BLOCKED_FILE_HOSTS = {"localhost", "localhost.localdomain"}
|
||
BLOCKED_FILE_HOST_SUFFIXES = (".localhost", ".local", ".internal")
|
||
SUPPORTED_UPLOAD_EXTENSIONS = tuple(
|
||
sorted({*SUPPORTED_EXCEL_EXTENSIONS, *SUPPORTED_ZIP_EXTENSIONS, ".zip"})
|
||
)
|
||
|
||
|
||
def _is_blocked_ip(address: str) -> bool:
|
||
ip = ipaddress.ip_address(address)
|
||
return (
|
||
ip.is_loopback
|
||
or ip.is_private
|
||
or ip.is_link_local
|
||
or ip.is_multicast
|
||
or ip.is_reserved
|
||
or ip.is_unspecified
|
||
)
|
||
|
||
|
||
def _validate_public_file_host(host: str) -> None:
|
||
normalized_host = host.strip("[]").rstrip(".").lower()
|
||
if normalized_host in BLOCKED_FILE_HOSTS or normalized_host.endswith(
|
||
BLOCKED_FILE_HOST_SUFFIXES
|
||
):
|
||
raise serializers.ValidationError("file_url host is not allowed")
|
||
|
||
try:
|
||
if _is_blocked_ip(normalized_host):
|
||
raise serializers.ValidationError("file_url IP address is not allowed")
|
||
return
|
||
except ValueError:
|
||
pass
|
||
|
||
try:
|
||
address_info = socket.getaddrinfo(
|
||
normalized_host,
|
||
443,
|
||
type=socket.SOCK_STREAM,
|
||
)
|
||
except socket.gaierror as exc:
|
||
raise serializers.ValidationError("file_url host cannot be resolved") from exc
|
||
|
||
for item in address_info:
|
||
if _is_blocked_ip(item[4][0]):
|
||
raise serializers.ValidationError("file_url host resolves to private IP")
|
||
|
||
|
||
# =============================================================================
|
||
# Минпромторг - Сертификаты промышленного производства
|
||
# =============================================================================
|
||
|
||
|
||
class CanonicalOrganizationListSerializer(serializers.ListSerializer):
|
||
"""Adds one canonical organization lookup for the current serialized page."""
|
||
|
||
def to_representation(self, data):
|
||
items = list(data.all() if hasattr(data, "all") else data)
|
||
field_sets = getattr(self.child, "canonical_organization_fields", ())
|
||
self.context[
|
||
ORGANIZATION_ENRICHMENT_CONTEXT_KEY
|
||
] = build_canonical_organization_enrichment(items, field_sets)
|
||
return super().to_representation(items)
|
||
|
||
|
||
class CanonicalOrganizationEnrichmentMixin:
|
||
"""Fill blank v1 organization fields while preserving existing contract."""
|
||
|
||
canonical_organization_fields: tuple[OrganizationFieldSet, ...] = ()
|
||
|
||
def to_representation(self, instance):
|
||
data = super().to_representation(instance)
|
||
if not self.canonical_organization_fields:
|
||
return data
|
||
|
||
enrichment = self.context.get(ORGANIZATION_ENRICHMENT_CONTEXT_KEY)
|
||
if enrichment is None:
|
||
enrichment = build_canonical_organization_enrichment(
|
||
[instance],
|
||
self.canonical_organization_fields,
|
||
)
|
||
return enrich_organization_fields(
|
||
data,
|
||
source=instance,
|
||
field_sets=self.canonical_organization_fields,
|
||
enrichment=enrichment,
|
||
)
|
||
|
||
|
||
class IndustrialCertificateSerializer(
|
||
CanonicalOrganizationEnrichmentMixin,
|
||
serializers.ModelSerializer,
|
||
):
|
||
"""
|
||
Сертификат промышленного производства РФ.
|
||
|
||
Данные загружаются из Минпромторга.
|
||
"""
|
||
|
||
canonical_organization_fields = (
|
||
OrganizationFieldSet(
|
||
name="organisation_name",
|
||
inn="inn",
|
||
ogrn="ogrn",
|
||
),
|
||
)
|
||
|
||
class Meta:
|
||
model = IndustrialCertificateRecord
|
||
list_serializer_class = CanonicalOrganizationListSerializer
|
||
fields = [
|
||
"id",
|
||
"load_batch",
|
||
"issue_date",
|
||
"issue_date_normalized",
|
||
"certificate_number",
|
||
"expiry_date",
|
||
"expiry_date_normalized",
|
||
"certificate_file_url",
|
||
"organisation_name",
|
||
"inn",
|
||
"ogrn",
|
||
"registry_organization",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
|
||
# =============================================================================
|
||
# Минпромторг - Реестр производителей
|
||
# =============================================================================
|
||
|
||
|
||
class ManufacturerSerializer(
|
||
CanonicalOrganizationEnrichmentMixin,
|
||
serializers.ModelSerializer,
|
||
):
|
||
"""
|
||
Производитель из реестра Минпромторга.
|
||
|
||
Данные загружаются из Минпромторга.
|
||
"""
|
||
|
||
canonical_organization_fields = (
|
||
OrganizationFieldSet(
|
||
name="full_legal_name",
|
||
inn="inn",
|
||
ogrn="ogrn",
|
||
),
|
||
)
|
||
|
||
class Meta:
|
||
model = ManufacturerRecord
|
||
list_serializer_class = CanonicalOrganizationListSerializer
|
||
fields = [
|
||
"id",
|
||
"load_batch",
|
||
"full_legal_name",
|
||
"inn",
|
||
"ogrn",
|
||
"address",
|
||
"registry_organization",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
|
||
class IndustrialProductSerializer(
|
||
CanonicalOrganizationEnrichmentMixin,
|
||
serializers.ModelSerializer,
|
||
):
|
||
"""
|
||
Промышленная продукция из реестра Минпромторга.
|
||
|
||
Данные загружаются из Минпромторга.
|
||
"""
|
||
|
||
canonical_organization_fields = (
|
||
OrganizationFieldSet(
|
||
name="full_organisation_name",
|
||
inn="inn",
|
||
ogrn="ogrn",
|
||
),
|
||
)
|
||
|
||
class Meta:
|
||
model = IndustrialProductRecord
|
||
list_serializer_class = CanonicalOrganizationListSerializer
|
||
fields = [
|
||
"id",
|
||
"load_batch",
|
||
"full_organisation_name",
|
||
"ogrn",
|
||
"inn",
|
||
"registry_number",
|
||
"product_name",
|
||
"product_model",
|
||
"okpd2_code",
|
||
"tnved_code",
|
||
"regulatory_document",
|
||
"registry_organization",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
|
||
# =============================================================================
|
||
# Единый реестр проверок (proverki.gov.ru)
|
||
# =============================================================================
|
||
|
||
|
||
class InspectionSerializer(
|
||
CanonicalOrganizationEnrichmentMixin,
|
||
serializers.ModelSerializer,
|
||
):
|
||
"""
|
||
Проверка из Единого реестра проверок.
|
||
|
||
Поддерживает ФЗ-294 и ФЗ-248.
|
||
"""
|
||
|
||
canonical_organization_fields = (
|
||
OrganizationFieldSet(
|
||
name="organisation_name",
|
||
inn="inn",
|
||
ogrn="ogrn",
|
||
),
|
||
)
|
||
|
||
class Meta:
|
||
model = InspectionRecord
|
||
list_serializer_class = CanonicalOrganizationListSerializer
|
||
fields = [
|
||
"id",
|
||
"load_batch",
|
||
"registration_number",
|
||
"inn",
|
||
"ogrn",
|
||
"organisation_name",
|
||
"control_authority",
|
||
"inspection_type",
|
||
"inspection_form",
|
||
"start_date",
|
||
"start_date_normalized",
|
||
"end_date",
|
||
"end_date_normalized",
|
||
"status",
|
||
"legal_basis",
|
||
"result",
|
||
"is_federal_law_248",
|
||
"data_year",
|
||
"data_month",
|
||
"registry_organization",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
|
||
# =============================================================================
|
||
# Государственные закупки (zakupki.gov.ru)
|
||
# =============================================================================
|
||
|
||
|
||
class ProcurementSerializer(
|
||
CanonicalOrganizationEnrichmentMixin,
|
||
serializers.ModelSerializer,
|
||
):
|
||
"""
|
||
Государственная закупка из ЕИС zakupki.gov.ru.
|
||
|
||
Поддерживает 44-ФЗ и 223-ФЗ.
|
||
"""
|
||
|
||
canonical_organization_fields = (
|
||
OrganizationFieldSet(
|
||
name="customer_name",
|
||
inn="customer_inn",
|
||
kpp="customer_kpp",
|
||
ogrn="customer_ogrn",
|
||
),
|
||
)
|
||
|
||
class Meta:
|
||
model = ProcurementRecord
|
||
list_serializer_class = CanonicalOrganizationListSerializer
|
||
fields = [
|
||
"id",
|
||
"load_batch",
|
||
"purchase_number",
|
||
"purchase_name",
|
||
"customer_inn",
|
||
"customer_kpp",
|
||
"customer_ogrn",
|
||
"customer_name",
|
||
"max_price",
|
||
"max_price_amount",
|
||
"currency_code",
|
||
"placement_method",
|
||
"publish_date",
|
||
"publish_date_normalized",
|
||
"end_date",
|
||
"end_date_normalized",
|
||
"status",
|
||
"law_type",
|
||
"purchase_object_info",
|
||
"href",
|
||
"region_code",
|
||
"data_year",
|
||
"data_month",
|
||
"registry_organization",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
|
||
# =============================================================================
|
||
# ФНС - Бухгалтерская отчетность
|
||
# =============================================================================
|
||
|
||
|
||
class FinancialReportLineSerializer(serializers.ModelSerializer):
|
||
"""Строка финансового отчета."""
|
||
|
||
class Meta:
|
||
model = FinancialReportLine
|
||
fields = [
|
||
"id",
|
||
"form_code",
|
||
"line_code",
|
||
"line_name",
|
||
"year",
|
||
"period_start",
|
||
"period_end",
|
||
]
|
||
|
||
|
||
class FinancialReportSerializer(serializers.ModelSerializer):
|
||
"""
|
||
Финансовый отчет ФНС.
|
||
|
||
Данные загружаются из Excel файлов.
|
||
"""
|
||
|
||
lines_count = serializers.SerializerMethodField()
|
||
|
||
class Meta:
|
||
model = FinancialReport
|
||
fields = [
|
||
"id",
|
||
"external_id",
|
||
"ogrn",
|
||
"registry_organization",
|
||
"file_name",
|
||
"file_hash",
|
||
"load_batch",
|
||
"status",
|
||
"source",
|
||
"error_message",
|
||
"created_at",
|
||
"updated_at",
|
||
"lines_count",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
def get_lines_count(self, obj) -> int:
|
||
return getattr(obj, "lines_count", obj.lines.count())
|
||
|
||
|
||
class FinancialReportDetailSerializer(FinancialReportSerializer):
|
||
"""Финансовый отчет с детализацией строк."""
|
||
|
||
lines = FinancialReportLineSerializer(many=True, read_only=True)
|
||
|
||
class Meta(FinancialReportSerializer.Meta):
|
||
fields = FinancialReportSerializer.Meta.fields + ["lines"]
|
||
|
||
|
||
class FNSFileUploadSerializer(serializers.Serializer):
|
||
"""
|
||
Сериализатор для загрузки файлов FNS.
|
||
|
||
Принимает список Excel файлов в формате fin_{id}_{ogrn}.xlsx
|
||
"""
|
||
|
||
file = serializers.FileField(
|
||
required=False,
|
||
help_text="Одиночный файл для загрузки (fin_*.xlsx)",
|
||
)
|
||
files = serializers.ListField(
|
||
child=serializers.FileField(),
|
||
required=False,
|
||
allow_empty=False,
|
||
help_text="Список файлов для загрузки (fin_*.xlsx)",
|
||
)
|
||
|
||
def validate(self, attrs):
|
||
files = attrs.get("files")
|
||
single_file = attrs.get("file")
|
||
|
||
if single_file and files:
|
||
raise serializers.ValidationError(
|
||
{"file": "Используйте либо file, либо files."}
|
||
)
|
||
if single_file:
|
||
files = [single_file]
|
||
if not files:
|
||
raise serializers.ValidationError(
|
||
{"file": "Нужно передать file или files."}
|
||
)
|
||
|
||
attrs["files"] = self._validate_uploaded_files(files)
|
||
return attrs
|
||
|
||
def _validate_uploaded_files(self, files):
|
||
"""Валидация файлов."""
|
||
for file in files:
|
||
if not FNS_XLSX_FILENAME_RE.match(file.name):
|
||
raise serializers.ValidationError(
|
||
f"Неверный формат имени файла: {file.name}. "
|
||
"Ожидается: fin_{{id}}_{{ogrn}}.xlsx"
|
||
)
|
||
|
||
return files
|
||
|
||
|
||
class FNSZipUploadSerializer(serializers.Serializer):
|
||
"""Сериализатор для загрузки ZIP архива с FNS Excel файлами."""
|
||
|
||
file = serializers.FileField(help_text="ZIP архив с файлами fin_*.xlsx")
|
||
|
||
def validate_file(self, value):
|
||
if not value.name.lower().endswith(".zip"):
|
||
raise serializers.ValidationError("Поддерживаются только ZIP архивы")
|
||
return value
|
||
|
||
|
||
class FNSServerPathUploadSerializer(serializers.Serializer):
|
||
"""Сериализатор для запуска обработки файла, уже лежащего на сервере."""
|
||
|
||
server_path = serializers.CharField(
|
||
help_text="Абсолютный путь к ZIP/XLSX внутри FNS watch directory",
|
||
)
|
||
|
||
|
||
class FNSFileUploadSuccessSerializer(serializers.Serializer):
|
||
"""Ответ одиночной загрузки FNS в формате frontend."""
|
||
|
||
success = serializers.BooleanField(read_only=True)
|
||
message = serializers.CharField(read_only=True)
|
||
|
||
|
||
class ParsingSettingsSerializer(serializers.ModelSerializer):
|
||
"""Настройки периодичности обновления источников парсинга."""
|
||
|
||
class Meta:
|
||
model = ParsingSettings
|
||
fields = [
|
||
"manufacturers_and_products",
|
||
"public_procurements",
|
||
"defense_unreliable_suppliers",
|
||
"planned_inspections",
|
||
"arbitration_cases",
|
||
"bankruptcy_procedures",
|
||
"information_security_registries",
|
||
]
|
||
|
||
|
||
class ParserSourceSerializer(serializers.Serializer):
|
||
"""Описание доступного parser source для dashboard/API."""
|
||
|
||
key = serializers.CharField()
|
||
source = serializers.CharField()
|
||
title = serializers.CharField()
|
||
agency = serializers.CharField()
|
||
data_scope = serializers.CharField()
|
||
task_name = serializers.CharField()
|
||
is_existing = serializers.BooleanField()
|
||
requires_file_url = serializers.BooleanField()
|
||
mode = serializers.CharField()
|
||
status = serializers.CharField()
|
||
owner = serializers.CharField(allow_blank=True)
|
||
upstream_url = serializers.URLField(allow_blank=True)
|
||
access_method = serializers.CharField()
|
||
parser_strategy = serializers.CharField()
|
||
source_notes = serializers.CharField(allow_blank=True)
|
||
supports_file_upload = serializers.BooleanField()
|
||
api_route = serializers.CharField(allow_blank=True)
|
||
result_list_url = serializers.CharField()
|
||
result_detail_url = serializers.CharField()
|
||
upload_url = serializers.CharField(allow_blank=True)
|
||
|
||
|
||
class ParserRunRequestSerializer(serializers.Serializer):
|
||
"""Параметры запуска Celery-задачи парсера."""
|
||
|
||
file_url = serializers.URLField(required=False, allow_blank=True)
|
||
file_path = serializers.CharField(required=False, allow_blank=True)
|
||
proxies = serializers.ListField(
|
||
child=serializers.CharField(),
|
||
required=False,
|
||
allow_empty=True,
|
||
)
|
||
vacancy_sources = serializers.ListField(
|
||
child=serializers.ChoiceField(choices=["trudvsem", "hh", "superjob"]),
|
||
required=False,
|
||
allow_empty=False,
|
||
)
|
||
registry_organizations_only = serializers.BooleanField(required=False)
|
||
registry_organization_limit = serializers.IntegerField(required=False, min_value=1)
|
||
year = serializers.IntegerField(required=False, min_value=2000, max_value=2100)
|
||
month = serializers.IntegerField(required=False, min_value=1, max_value=12)
|
||
limit = serializers.IntegerField(required=False, min_value=1, max_value=1000)
|
||
offset = serializers.IntegerField(required=False, min_value=0)
|
||
max_months_per_law = serializers.IntegerField(
|
||
required=False,
|
||
min_value=1,
|
||
max_value=36,
|
||
)
|
||
start_year = serializers.IntegerField(
|
||
required=False,
|
||
min_value=2000,
|
||
max_value=2100,
|
||
)
|
||
start_month = serializers.IntegerField(required=False, min_value=1, max_value=12)
|
||
include_fz294 = serializers.BooleanField(required=False)
|
||
include_fz248 = serializers.BooleanField(required=False)
|
||
current_year = serializers.IntegerField(
|
||
required=False,
|
||
min_value=2000,
|
||
max_value=2100,
|
||
)
|
||
current_month = serializers.IntegerField(required=False, min_value=1, max_value=12)
|
||
region_code = serializers.CharField(required=False, allow_blank=True)
|
||
law_type = serializers.CharField(required=False, allow_blank=True)
|
||
company_inn = serializers.CharField(required=False, allow_blank=True)
|
||
text = serializers.CharField(required=False, allow_blank=True)
|
||
|
||
def validate_file_url(self, value: str) -> str:
|
||
"""Разрешить worker-download только с публичных HTTPS URL."""
|
||
if not value:
|
||
return value
|
||
|
||
parsed = urlsplit(value)
|
||
if parsed.scheme != "https":
|
||
raise serializers.ValidationError("file_url must use https")
|
||
if parsed.username or parsed.password:
|
||
raise serializers.ValidationError("file_url credentials are not allowed")
|
||
if not parsed.hostname:
|
||
raise serializers.ValidationError("file_url host is required")
|
||
|
||
_validate_public_file_host(parsed.hostname)
|
||
return value
|
||
|
||
def validate(self, attrs):
|
||
attrs = super().validate(attrs)
|
||
if ("start_year" in attrs) != ("start_month" in attrs):
|
||
raise serializers.ValidationError(
|
||
{
|
||
"start_month": (
|
||
"start_year and start_month must be provided together"
|
||
)
|
||
}
|
||
)
|
||
return attrs
|
||
|
||
|
||
class ParserUploadRequestSerializer(serializers.Serializer):
|
||
"""Файл ручной загрузки реестра или финансовой выгрузки."""
|
||
|
||
file = serializers.FileField()
|
||
|
||
def validate_file(self, value):
|
||
file_name = (value.name or "").lower()
|
||
if not file_name.endswith(SUPPORTED_UPLOAD_EXTENSIONS):
|
||
extensions = ", ".join(SUPPORTED_UPLOAD_EXTENSIONS)
|
||
raise serializers.ValidationError(
|
||
f"Unsupported file extension. Allowed: {extensions}"
|
||
)
|
||
if value.size > MAX_FILE_SIZE_BYTES:
|
||
raise serializers.ValidationError(
|
||
f"File exceeds size limit: {MAX_FILE_SIZE_BYTES} bytes"
|
||
)
|
||
return value
|
||
|
||
|
||
class ParserScheduleRequestSerializer(ParserRunRequestSerializer):
|
||
"""Параметры создания/обновления периодической Celery-задачи парсера."""
|
||
|
||
SCHEDULE_TYPES = ("interval", "crontab")
|
||
PERIODS = ("seconds", "minutes", "hours", "days")
|
||
|
||
source_key = serializers.CharField(required=False)
|
||
name = serializers.CharField(required=False, allow_blank=True, max_length=200)
|
||
enabled = serializers.BooleanField(required=False, default=True)
|
||
schedule_type = serializers.ChoiceField(
|
||
choices=SCHEDULE_TYPES,
|
||
required=False,
|
||
default="interval",
|
||
)
|
||
every = serializers.IntegerField(required=False, min_value=1)
|
||
period = serializers.ChoiceField(choices=PERIODS, required=False, default="hours")
|
||
minute = serializers.CharField(required=False, allow_blank=True, default="0")
|
||
hour = serializers.CharField(required=False, allow_blank=True, default="*")
|
||
day_of_week = serializers.CharField(required=False, allow_blank=True, default="*")
|
||
day_of_month = serializers.CharField(required=False, allow_blank=True, default="*")
|
||
month_of_year = serializers.CharField(required=False, allow_blank=True, default="*")
|
||
|
||
def validate(self, attrs):
|
||
attrs = super().validate(attrs)
|
||
if attrs.get("schedule_type", "interval") == "interval" and not attrs.get(
|
||
"every"
|
||
):
|
||
raise serializers.ValidationError(
|
||
{"every": "Required for interval schedule"}
|
||
)
|
||
return attrs
|
||
|
||
|
||
class ParserScheduleSerializer(serializers.Serializer):
|
||
"""Описание периодической задачи парсера."""
|
||
|
||
id = serializers.IntegerField()
|
||
name = serializers.CharField()
|
||
source_key = serializers.CharField()
|
||
source = serializers.CharField()
|
||
title = serializers.CharField()
|
||
task_name = serializers.CharField()
|
||
enabled = serializers.BooleanField()
|
||
schedule_type = serializers.CharField()
|
||
schedule = serializers.DictField()
|
||
params = serializers.DictField()
|
||
last_run_at = serializers.DateTimeField(allow_null=True)
|
||
total_run_count = serializers.IntegerField()
|
||
date_changed = serializers.DateTimeField()
|
||
|
||
|
||
class ParserListQuerySerializer(serializers.Serializer):
|
||
"""Общие query-параметры списков парсеров."""
|
||
|
||
limit = serializers.IntegerField(
|
||
required=False,
|
||
default=50,
|
||
min_value=1,
|
||
max_value=200,
|
||
)
|
||
|
||
|
||
class ParserResultQuerySerializer(serializers.Serializer):
|
||
"""Query-параметры per-source result endpoints."""
|
||
|
||
page = serializers.IntegerField(required=False, default=1, min_value=1)
|
||
page_size = serializers.IntegerField(
|
||
required=False,
|
||
default=20,
|
||
min_value=1,
|
||
max_value=100,
|
||
)
|
||
limit = serializers.IntegerField(required=False, min_value=1, max_value=100)
|
||
id = serializers.IntegerField(required=False, min_value=1)
|
||
source = serializers.CharField(required=False, allow_blank=True)
|
||
external_id = serializers.CharField(required=False, allow_blank=True)
|
||
inn = serializers.CharField(required=False, allow_blank=True)
|
||
ogrn = serializers.CharField(required=False, allow_blank=True)
|
||
load_batch = serializers.IntegerField(required=False, min_value=1)
|
||
batch_id = serializers.IntegerField(required=False, min_value=1)
|
||
status = serializers.CharField(required=False, allow_blank=True)
|
||
record_date = serializers.CharField(required=False, allow_blank=True)
|
||
search = serializers.CharField(required=False, allow_blank=True)
|
||
ordering = serializers.CharField(required=False, allow_blank=True)
|
||
include_payload = serializers.BooleanField(required=False, default=True)
|
||
|
||
def validate(self, attrs):
|
||
attrs = super().validate(attrs)
|
||
if attrs.get("limit"):
|
||
attrs["page"] = 1
|
||
attrs["page_size"] = attrs["limit"]
|
||
if attrs.get("batch_id") and not attrs.get("load_batch"):
|
||
attrs["load_batch"] = attrs["batch_id"]
|
||
return attrs
|
||
|
||
|
||
class ParserRunResponseSerializer(serializers.Serializer):
|
||
"""Ответ API на запуск задачи."""
|
||
|
||
task_id = serializers.CharField()
|
||
source = serializers.CharField()
|
||
task_name = serializers.CharField()
|
||
|
||
|
||
# =============================================================================
|
||
# Служебные модели
|
||
# =============================================================================
|
||
|
||
|
||
class ParserLoadLogSerializer(serializers.ModelSerializer):
|
||
"""
|
||
Лог загрузки парсера.
|
||
|
||
Информация о каждой загрузке данных из внешнего источника.
|
||
"""
|
||
|
||
source_display = serializers.CharField(source="get_source_display", read_only=True)
|
||
organizations_count = serializers.SerializerMethodField()
|
||
|
||
class Meta:
|
||
model = ParserLoadLog
|
||
fields = [
|
||
"id",
|
||
"batch_id",
|
||
"source",
|
||
"source_display",
|
||
"records_count",
|
||
"organizations_count",
|
||
"status",
|
||
"error_message",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
def get_organizations_count(self, obj) -> int:
|
||
if obj.source == ParserLoadLog.Source.FNS_REPORTS:
|
||
return (
|
||
FinancialReport.objects.filter(load_batch=obj.batch_id)
|
||
.exclude(ogrn="")
|
||
.values("ogrn")
|
||
.distinct()
|
||
.count()
|
||
)
|
||
if obj.source == ParserLoadLog.Source.INDUSTRIAL:
|
||
return (
|
||
IndustrialCertificateRecord.objects.filter(load_batch=obj.batch_id)
|
||
.exclude(inn="")
|
||
.values("inn")
|
||
.distinct()
|
||
.count()
|
||
)
|
||
if obj.source == ParserLoadLog.Source.INDUSTRIAL_PRODUCTS:
|
||
return (
|
||
IndustrialProductRecord.objects.filter(load_batch=obj.batch_id)
|
||
.exclude(inn="")
|
||
.values("inn")
|
||
.distinct()
|
||
.count()
|
||
)
|
||
if obj.source == ParserLoadLog.Source.MANUFACTURES:
|
||
return (
|
||
ManufacturerRecord.objects.filter(load_batch=obj.batch_id)
|
||
.exclude(inn="")
|
||
.values("inn")
|
||
.distinct()
|
||
.count()
|
||
)
|
||
if obj.source == ParserLoadLog.Source.INSPECTIONS:
|
||
return (
|
||
InspectionRecord.objects.filter(load_batch=obj.batch_id)
|
||
.exclude(inn="")
|
||
.values("inn")
|
||
.distinct()
|
||
.count()
|
||
)
|
||
if obj.source == ParserLoadLog.Source.PROCUREMENTS:
|
||
return (
|
||
ProcurementRecord.objects.filter(load_batch=obj.batch_id)
|
||
.exclude(customer_inn="")
|
||
.values("customer_inn")
|
||
.distinct()
|
||
.count()
|
||
)
|
||
return 0
|
||
|
||
|
||
class GenericParserRecordSerializer(
|
||
CanonicalOrganizationEnrichmentMixin,
|
||
serializers.ModelSerializer,
|
||
):
|
||
"""Сериализатор универсальных записей новых источников."""
|
||
|
||
canonical_organization_fields = (
|
||
OrganizationFieldSet(
|
||
name="organisation_name",
|
||
inn="inn",
|
||
ogrn="ogrn",
|
||
),
|
||
)
|
||
|
||
class Meta:
|
||
model = GenericParserRecord
|
||
list_serializer_class = CanonicalOrganizationListSerializer
|
||
fields = [
|
||
"id",
|
||
"load_batch",
|
||
"source",
|
||
"external_id",
|
||
"inn",
|
||
"ogrn",
|
||
"organisation_name",
|
||
"title",
|
||
"record_date",
|
||
"amount",
|
||
"status",
|
||
"url",
|
||
"payload",
|
||
"registry_organization",
|
||
"created_at",
|
||
"updated_at",
|
||
]
|
||
read_only_fields = fields
|
||
|
||
|
||
class ParserResultRecordSerializer(serializers.Serializer):
|
||
"""Унифицированная запись результата конкретного источника."""
|
||
|
||
id = serializers.IntegerField()
|
||
load_batch = serializers.IntegerField()
|
||
source = serializers.CharField()
|
||
vacancy_source = serializers.CharField(allow_blank=True, required=False)
|
||
external_id = serializers.CharField(allow_blank=True)
|
||
inn = serializers.CharField(allow_blank=True)
|
||
ogrn = serializers.CharField(allow_blank=True)
|
||
organisation_name = serializers.CharField(allow_blank=True)
|
||
title = serializers.CharField(allow_blank=True)
|
||
record_date = serializers.CharField(allow_blank=True)
|
||
amount = serializers.DecimalField(
|
||
max_digits=20,
|
||
decimal_places=2,
|
||
allow_null=True,
|
||
)
|
||
status = serializers.CharField(allow_blank=True)
|
||
url = serializers.CharField(allow_blank=True)
|
||
payload = serializers.DictField()
|
||
created_at = serializers.DateTimeField()
|
||
updated_at = serializers.DateTimeField()
|
||
|
||
|
||
class ParserLoadLogListSerializer(serializers.Serializer):
|
||
"""Строка списка логов в frontend-friendly формате."""
|
||
|
||
id = serializers.IntegerField(read_only=True)
|
||
batch_id = serializers.IntegerField(read_only=True)
|
||
source = serializers.CharField(read_only=True)
|
||
source_label = serializers.CharField(read_only=True, allow_null=True)
|
||
records_count = serializers.IntegerField(read_only=True)
|
||
organizations_count = serializers.IntegerField(read_only=True)
|
||
status = serializers.CharField(read_only=True)
|
||
status_label = serializers.CharField(read_only=True)
|
||
error_message = serializers.CharField(read_only=True, allow_blank=True)
|
||
created_at = serializers.DateTimeField(read_only=True)
|
||
updated_at = serializers.DateTimeField(read_only=True)
|
||
|
||
|
||
class ParserLoadLogPageSerializer(serializers.Serializer):
|
||
"""Пагинированный ответ списка логов для frontend."""
|
||
|
||
count = serializers.IntegerField(read_only=True)
|
||
next = serializers.CharField(read_only=True, allow_null=True)
|
||
previous = serializers.CharField(read_only=True, allow_null=True)
|
||
results = ParserLoadLogListSerializer(many=True, read_only=True)
|
||
|
||
|
||
class SourceCardRefreshParamSerializer(serializers.Serializer):
|
||
"""Описание параметра ручного обновления карточки источника."""
|
||
|
||
name = serializers.CharField(read_only=True)
|
||
label = serializers.CharField(read_only=True)
|
||
description = serializers.CharField(read_only=True)
|
||
required = serializers.BooleanField(read_only=True)
|
||
type = serializers.CharField(read_only=True)
|
||
default = serializers.JSONField(read_only=True)
|
||
|
||
|
||
class SourceCardLoadSerializer(serializers.Serializer):
|
||
"""Сводка по последней загрузке источника."""
|
||
|
||
batch_id = serializers.IntegerField(read_only=True)
|
||
source = serializers.CharField(read_only=True)
|
||
source_display = serializers.CharField(read_only=True)
|
||
records_count = serializers.IntegerField(read_only=True)
|
||
status = serializers.CharField(read_only=True)
|
||
error_message = serializers.CharField(read_only=True)
|
||
created_at = serializers.DateTimeField(read_only=True)
|
||
updated_at = serializers.DateTimeField(read_only=True)
|
||
|
||
|
||
class SourceCardTaskSerializer(serializers.Serializer):
|
||
"""Краткая информация по фоновой задаче карточки."""
|
||
|
||
task_id = serializers.CharField(read_only=True)
|
||
task_name = serializers.CharField(read_only=True)
|
||
status = serializers.CharField(read_only=True)
|
||
progress = serializers.IntegerField(read_only=True)
|
||
progress_message = serializers.CharField(read_only=True)
|
||
started_at = serializers.DateTimeField(read_only=True)
|
||
created_at = serializers.DateTimeField(read_only=True)
|
||
meta = serializers.JSONField(read_only=True)
|
||
|
||
|
||
class SourceCardItemSerializer(serializers.Serializer):
|
||
"""Подисточник внутри агрегированной карточки."""
|
||
|
||
code = serializers.CharField(read_only=True)
|
||
title = serializers.CharField(read_only=True)
|
||
description = serializers.CharField(read_only=True)
|
||
parser_source = serializers.CharField(read_only=True, allow_null=True)
|
||
parser_source_display = serializers.CharField(read_only=True, allow_null=True)
|
||
records_count = serializers.IntegerField(read_only=True)
|
||
organizations_count = serializers.IntegerField(read_only=True)
|
||
last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||
|
||
|
||
class SourceCardSerializer(serializers.Serializer):
|
||
"""Serializer for frontend source cards list."""
|
||
|
||
slug = serializers.CharField(read_only=True)
|
||
title = serializers.CharField(read_only=True)
|
||
description = serializers.CharField(read_only=True)
|
||
order = serializers.IntegerField(read_only=True)
|
||
is_available = serializers.BooleanField(read_only=True)
|
||
status = serializers.CharField(read_only=True)
|
||
status_label = serializers.CharField(read_only=True)
|
||
progress = serializers.IntegerField(read_only=True)
|
||
records_count = serializers.IntegerField(read_only=True)
|
||
organizations_count = serializers.IntegerField(read_only=True)
|
||
last_updated_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||
next_update_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||
error_message = serializers.CharField(read_only=True)
|
||
task_names = serializers.ListField(child=serializers.CharField(), read_only=True)
|
||
refresh_requires_params = serializers.BooleanField(read_only=True)
|
||
refresh_params = SourceCardRefreshParamSerializer(many=True, read_only=True)
|
||
|
||
|
||
class SourceTaskStatusSerializer(serializers.Serializer):
|
||
"""Табличная строка статуса источника для экрана задач парсинга."""
|
||
|
||
row_number = serializers.IntegerField(read_only=True)
|
||
slug = serializers.CharField(read_only=True)
|
||
source = serializers.CharField(read_only=True)
|
||
status = serializers.CharField(read_only=True)
|
||
status_label = serializers.CharField(read_only=True)
|
||
actualized_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||
next_update_at = serializers.DateTimeField(read_only=True, allow_null=True)
|
||
records_count = serializers.IntegerField(read_only=True)
|
||
organizations_count = serializers.IntegerField(read_only=True)
|
||
progress = serializers.IntegerField(read_only=True)
|
||
error_message = serializers.CharField(read_only=True)
|
||
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
|
||
|
||
|
||
class SourceCardDetailSerializer(SourceCardSerializer):
|
||
"""Detailed serializer for a single frontend source card."""
|
||
|
||
active_tasks = SourceCardTaskSerializer(many=True, read_only=True)
|
||
source_items = SourceCardItemSerializer(many=True, read_only=True)
|
||
latest_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||
latest_success_load = SourceCardLoadSerializer(read_only=True, allow_null=True)
|
||
|
||
|
||
class SourceCardRefreshRequestSerializer(serializers.Serializer):
|
||
"""Request body for manual card refresh."""
|
||
|
||
params = serializers.DictField(required=False)
|
||
|
||
|
||
class SourceCardRefreshTaskSerializer(serializers.Serializer):
|
||
"""Queued task info after refresh start."""
|
||
|
||
task_id = serializers.CharField(read_only=True)
|
||
task_name = serializers.CharField(read_only=True)
|
||
|
||
|
||
class SourceCardRefreshResponseSerializer(serializers.Serializer):
|
||
"""Response serializer for manual card refresh."""
|
||
|
||
source_card = serializers.CharField(read_only=True)
|
||
status = serializers.CharField(read_only=True)
|
||
requested_at = serializers.DateTimeField(read_only=True)
|
||
tasks = SourceCardRefreshTaskSerializer(many=True, read_only=True)
|
||
|
||
|
||
class FrontendApiResponseSerializer(serializers.Serializer):
|
||
"""Общая форма success/data/errors/meta для frontend sources API."""
|
||
|
||
success = serializers.BooleanField(read_only=True)
|
||
errors = serializers.JSONField(read_only=True, allow_null=True)
|
||
meta = serializers.JSONField(read_only=True, allow_null=True)
|
||
|
||
|
||
class SourceCardListResponseSerializer(FrontendApiResponseSerializer):
|
||
"""Envelope для списка карточек источников."""
|
||
|
||
data = SourceCardSerializer(many=True, read_only=True)
|
||
|
||
|
||
class SourceTaskStatusListResponseSerializer(FrontendApiResponseSerializer):
|
||
"""Envelope для списка статусов источников."""
|
||
|
||
data = SourceTaskStatusSerializer(many=True, read_only=True)
|
||
|
||
|
||
class SourceCardDetailResponseSerializer(FrontendApiResponseSerializer):
|
||
"""Envelope для детальной карточки источника."""
|
||
|
||
data = SourceCardDetailSerializer(read_only=True)
|
||
|
||
|
||
class SourceCardRefreshFrontendResponseSerializer(serializers.Serializer):
|
||
"""Минимальный ответ запуска обновления карточки по md."""
|
||
|
||
task_id = serializers.CharField(read_only=True, allow_null=True)
|
||
status = serializers.CharField(read_only=True)
|