feat: add parser source dashboard and scheduling

src/apps/parsers/serializers.py

@@ -1,13 +1,19 @@
-"""
-Сериализаторы для приложения парсеров.
+"""Сериализаторы для приложения парсеров."""
 
-Все сериализаторы read-only, так как данные загружаются только через парсеры.
-"""
+import ipaddress
+import socket
+from urllib.parse import urlsplit
 
+from apps.parsers.clients.common.structured import (
+    MAX_FILE_SIZE_BYTES,
+    SUPPORTED_EXCEL_EXTENSIONS,
+    SUPPORTED_ZIP_EXTENSIONS,
+)
 from apps.parsers.fns_upload import FNS_XLSX_FILENAME_RE
 from apps.parsers.models import (
     FinancialReport,
     FinancialReportLine,
+    GenericParserRecord,
     IndustrialCertificateRecord,
     IndustrialProductRecord,
     InspectionRecord,
@@ -18,6 +24,53 @@ from apps.parsers.models import (
 )
 from rest_framework import serializers
 
+BLOCKED_FILE_HOSTS = {"localhost", "localhost.localdomain"}
+BLOCKED_FILE_HOST_SUFFIXES = (".localhost", ".local", ".internal")
+SUPPORTED_UPLOAD_EXTENSIONS = tuple(
+    sorted({*SUPPORTED_EXCEL_EXTENSIONS, *SUPPORTED_ZIP_EXTENSIONS, ".zip"})
+)
+
+
+def _is_blocked_ip(address: str) -> bool:
+    ip = ipaddress.ip_address(address)
+    return (
+        ip.is_loopback
+        or ip.is_private
+        or ip.is_link_local
+        or ip.is_multicast
+        or ip.is_reserved
+        or ip.is_unspecified
+    )
+
+
+def _validate_public_file_host(host: str) -> None:
+    normalized_host = host.strip("[]").rstrip(".").lower()
+    if normalized_host in BLOCKED_FILE_HOSTS or normalized_host.endswith(
+        BLOCKED_FILE_HOST_SUFFIXES
+    ):
+        raise serializers.ValidationError("file_url host is not allowed")
+
+    try:
+        if _is_blocked_ip(normalized_host):
+            raise serializers.ValidationError("file_url IP address is not allowed")
+        return
+    except ValueError:
+        pass
+
+    try:
+        address_info = socket.getaddrinfo(
+            normalized_host,
+            443,
+            type=socket.SOCK_STREAM,
+        )
+    except socket.gaierror as exc:
+        raise serializers.ValidationError("file_url host cannot be resolved") from exc
+
+    for item in address_info:
+        if _is_blocked_ip(item[4][0]):
+            raise serializers.ValidationError("file_url host resolves to private IP")
+
+
 # =============================================================================
 # Минпромторг - Сертификаты промышленного производства
 # =============================================================================
@@ -337,6 +390,220 @@ class ParsingSettingsSerializer(serializers.ModelSerializer):
         ]
 
 
+class ParserSourceSerializer(serializers.Serializer):
+    """Описание доступного parser source для dashboard/API."""
+
+    key = serializers.CharField()
+    source = serializers.CharField()
+    title = serializers.CharField()
+    agency = serializers.CharField()
+    data_scope = serializers.CharField()
+    task_name = serializers.CharField()
+    is_existing = serializers.BooleanField()
+    requires_file_url = serializers.BooleanField()
+    mode = serializers.CharField()
+    status = serializers.CharField()
+    owner = serializers.CharField(allow_blank=True)
+    upstream_url = serializers.URLField(allow_blank=True)
+    access_method = serializers.CharField()
+    parser_strategy = serializers.CharField()
+    source_notes = serializers.CharField(allow_blank=True)
+    supports_file_upload = serializers.BooleanField()
+    result_list_url = serializers.CharField()
+    result_detail_url = serializers.CharField()
+    upload_url = serializers.CharField(allow_blank=True)
+
+
+class ParserRunRequestSerializer(serializers.Serializer):
+    """Параметры запуска Celery-задачи парсера."""
+
+    file_url = serializers.URLField(required=False, allow_blank=True)
+    file_path = serializers.CharField(required=False, allow_blank=True)
+    proxies = serializers.ListField(
+        child=serializers.CharField(),
+        required=False,
+        allow_empty=True,
+    )
+    year = serializers.IntegerField(required=False, min_value=2000, max_value=2100)
+    month = serializers.IntegerField(required=False, min_value=1, max_value=12)
+    limit = serializers.IntegerField(required=False, min_value=1, max_value=1000)
+    offset = serializers.IntegerField(required=False, min_value=0)
+    max_months_per_law = serializers.IntegerField(
+        required=False,
+        min_value=1,
+        max_value=36,
+    )
+    start_year = serializers.IntegerField(
+        required=False,
+        min_value=2000,
+        max_value=2100,
+    )
+    start_month = serializers.IntegerField(required=False, min_value=1, max_value=12)
+    include_fz294 = serializers.BooleanField(required=False)
+    include_fz248 = serializers.BooleanField(required=False)
+    current_year = serializers.IntegerField(
+        required=False,
+        min_value=2000,
+        max_value=2100,
+    )
+    current_month = serializers.IntegerField(required=False, min_value=1, max_value=12)
+    region_code = serializers.CharField(required=False, allow_blank=True)
+    law_type = serializers.CharField(required=False, allow_blank=True)
+    company_inn = serializers.CharField(required=False, allow_blank=True)
+    text = serializers.CharField(required=False, allow_blank=True)
+
+    def validate_file_url(self, value: str) -> str:
+        """Разрешить worker-download только с публичных HTTPS URL."""
+        if not value:
+            return value
+
+        parsed = urlsplit(value)
+        if parsed.scheme != "https":
+            raise serializers.ValidationError("file_url must use https")
+        if parsed.username or parsed.password:
+            raise serializers.ValidationError("file_url credentials are not allowed")
+        if not parsed.hostname:
+            raise serializers.ValidationError("file_url host is required")
+
+        _validate_public_file_host(parsed.hostname)
+        return value
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+        if ("start_year" in attrs) != ("start_month" in attrs):
+            raise serializers.ValidationError(
+                {
+                    "start_month": (
+                        "start_year and start_month must be provided together"
+                    )
+                }
+            )
+        return attrs
+
+
+class ParserUploadRequestSerializer(serializers.Serializer):
+    """Файл ручной загрузки реестра или финансовой выгрузки."""
+
+    file = serializers.FileField()
+
+    def validate_file(self, value):
+        file_name = (value.name or "").lower()
+        if not file_name.endswith(SUPPORTED_UPLOAD_EXTENSIONS):
+            extensions = ", ".join(SUPPORTED_UPLOAD_EXTENSIONS)
+            raise serializers.ValidationError(
+                f"Unsupported file extension. Allowed: {extensions}"
+            )
+        if value.size > MAX_FILE_SIZE_BYTES:
+            raise serializers.ValidationError(
+                f"File exceeds size limit: {MAX_FILE_SIZE_BYTES} bytes"
+            )
+        return value
+
+
+class ParserScheduleRequestSerializer(ParserRunRequestSerializer):
+    """Параметры создания/обновления периодической Celery-задачи парсера."""
+
+    SCHEDULE_TYPES = ("interval", "crontab")
+    PERIODS = ("seconds", "minutes", "hours", "days")
+
+    source_key = serializers.CharField(required=False)
+    name = serializers.CharField(required=False, allow_blank=True, max_length=200)
+    enabled = serializers.BooleanField(required=False, default=True)
+    schedule_type = serializers.ChoiceField(
+        choices=SCHEDULE_TYPES,
+        required=False,
+        default="interval",
+    )
+    every = serializers.IntegerField(required=False, min_value=1)
+    period = serializers.ChoiceField(choices=PERIODS, required=False, default="hours")
+    minute = serializers.CharField(required=False, allow_blank=True, default="0")
+    hour = serializers.CharField(required=False, allow_blank=True, default="*")
+    day_of_week = serializers.CharField(required=False, allow_blank=True, default="*")
+    day_of_month = serializers.CharField(required=False, allow_blank=True, default="*")
+    month_of_year = serializers.CharField(required=False, allow_blank=True, default="*")
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+        if attrs.get("schedule_type", "interval") == "interval" and not attrs.get(
+            "every"
+        ):
+            raise serializers.ValidationError(
+                {"every": "Required for interval schedule"}
+            )
+        return attrs
+
+
+class ParserScheduleSerializer(serializers.Serializer):
+    """Описание периодической задачи парсера."""
+
+    id = serializers.IntegerField()
+    name = serializers.CharField()
+    source_key = serializers.CharField()
+    source = serializers.CharField()
+    title = serializers.CharField()
+    task_name = serializers.CharField()
+    enabled = serializers.BooleanField()
+    schedule_type = serializers.CharField()
+    schedule = serializers.DictField()
+    params = serializers.DictField()
+    last_run_at = serializers.DateTimeField(allow_null=True)
+    total_run_count = serializers.IntegerField()
+    date_changed = serializers.DateTimeField()
+
+
+class ParserListQuerySerializer(serializers.Serializer):
+    """Общие query-параметры списков парсеров."""
+
+    limit = serializers.IntegerField(
+        required=False,
+        default=50,
+        min_value=1,
+        max_value=200,
+    )
+
+
+class ParserResultQuerySerializer(serializers.Serializer):
+    """Query-параметры per-source result endpoints."""
+
+    page = serializers.IntegerField(required=False, default=1, min_value=1)
+    page_size = serializers.IntegerField(
+        required=False,
+        default=20,
+        min_value=1,
+        max_value=100,
+    )
+    limit = serializers.IntegerField(required=False, min_value=1, max_value=100)
+    id = serializers.IntegerField(required=False, min_value=1)
+    source = serializers.CharField(required=False, allow_blank=True)
+    external_id = serializers.CharField(required=False, allow_blank=True)
+    inn = serializers.CharField(required=False, allow_blank=True)
+    ogrn = serializers.CharField(required=False, allow_blank=True)
+    load_batch = serializers.IntegerField(required=False, min_value=1)
+    batch_id = serializers.IntegerField(required=False, min_value=1)
+    status = serializers.CharField(required=False, allow_blank=True)
+    record_date = serializers.CharField(required=False, allow_blank=True)
+    search = serializers.CharField(required=False, allow_blank=True)
+    ordering = serializers.CharField(required=False, allow_blank=True)
+    include_payload = serializers.BooleanField(required=False, default=True)
+
+    def validate(self, attrs):
+        attrs = super().validate(attrs)
+        if attrs.get("limit"):
+            attrs["page"] = 1
+            attrs["page_size"] = attrs["limit"]
+        if attrs.get("batch_id") and not attrs.get("load_batch"):
+            attrs["load_batch"] = attrs["batch_id"]
+        return attrs
+
+
+class ParserRunResponseSerializer(serializers.Serializer):
+    """Ответ API на запуск задачи."""
+
+    task_id = serializers.CharField()
+    source = serializers.CharField()
+    task_name = serializers.CharField()
+
+
 # =============================================================================
 # Служебные модели
 # =============================================================================
@@ -420,6 +687,56 @@ class ParserLoadLogSerializer(serializers.ModelSerializer):
         return 0
 
 
+class GenericParserRecordSerializer(serializers.ModelSerializer):
+    """Сериализатор универсальных записей новых источников."""
+
+    class Meta:
+        model = GenericParserRecord
+        fields = [
+            "id",
+            "load_batch",
+            "source",
+            "external_id",
+            "inn",
+            "ogrn",
+            "organisation_name",
+            "title",
+            "record_date",
+            "amount",
+            "status",
+            "url",
+            "payload",
+            "registry_organization",
+            "created_at",
+            "updated_at",
+        ]
+        read_only_fields = fields
+
+
+class ParserResultRecordSerializer(serializers.Serializer):
+    """Унифицированная запись результата конкретного источника."""
+
+    id = serializers.IntegerField()
+    load_batch = serializers.IntegerField()
+    source = serializers.CharField()
+    external_id = serializers.CharField(allow_blank=True)
+    inn = serializers.CharField(allow_blank=True)
+    ogrn = serializers.CharField(allow_blank=True)
+    organisation_name = serializers.CharField(allow_blank=True)
+    title = serializers.CharField(allow_blank=True)
+    record_date = serializers.CharField(allow_blank=True)
+    amount = serializers.DecimalField(
+        max_digits=20,
+        decimal_places=2,
+        allow_null=True,
+    )
+    status = serializers.CharField(allow_blank=True)
+    url = serializers.CharField(allow_blank=True)
+    payload = serializers.DictField()
+    created_at = serializers.DateTimeField()
+    updated_at = serializers.DateTimeField()
+
+
 class ParserLoadLogListSerializer(serializers.Serializer):
     """Строка списка логов в frontend-friendly формате."""