diff --git a/docker/scripts/start-web.sh b/docker/scripts/start-web.sh index 5b32530..047d5fa 100755 --- a/docker/scripts/start-web.sh +++ b/docker/scripts/start-web.sh @@ -9,6 +9,8 @@ case "${DJANGO_SETTINGS_MODULE}" in ;; esac +python src/manage.py collectstatic --noinput + exec gunicorn core.wsgi:application \ --bind "0.0.0.0:${PORT:-8000}" \ --workers "${GUNICORN_WORKERS:-3}" \ diff --git a/input/demo-periods/Ф-1_2022-12.xlsx b/input/demo-periods/Ф-1_2022-12.xlsx new file mode 100644 index 0000000..104d661 Binary files /dev/null and b/input/demo-periods/Ф-1_2022-12.xlsx differ diff --git a/input/demo-periods/Ф-1_2023-12.xlsx b/input/demo-periods/Ф-1_2023-12.xlsx new file mode 100644 index 0000000..d573aa4 Binary files /dev/null and b/input/demo-periods/Ф-1_2023-12.xlsx differ diff --git a/input/demo-periods/Ф-1_2024-12.xlsx b/input/demo-periods/Ф-1_2024-12.xlsx new file mode 100644 index 0000000..7e6973c Binary files /dev/null and b/input/demo-periods/Ф-1_2024-12.xlsx differ diff --git a/input/demo-periods/Ф-1_2025-09.xlsx b/input/demo-periods/Ф-1_2025-09.xlsx new file mode 100644 index 0000000..cd48307 Binary files /dev/null and b/input/demo-periods/Ф-1_2025-09.xlsx differ diff --git a/input/demo-periods/Ф-1_2025-12.xlsx b/input/demo-periods/Ф-1_2025-12.xlsx new file mode 100644 index 0000000..139cc30 Binary files /dev/null and b/input/demo-periods/Ф-1_2025-12.xlsx differ diff --git a/input/demo-periods/Ф-1_2026-03.xlsx b/input/demo-periods/Ф-1_2026-03.xlsx new file mode 100644 index 0000000..c4bcd0e Binary files /dev/null and b/input/demo-periods/Ф-1_2026-03.xlsx differ diff --git a/input/demo-periods/Ф-1_2026-06.xlsx b/input/demo-periods/Ф-1_2026-06.xlsx new file mode 100644 index 0000000..9d77c6f Binary files /dev/null and b/input/demo-periods/Ф-1_2026-06.xlsx differ diff --git a/input/demo-periods/Ф-2_2025-Q3.xlsx b/input/demo-periods/Ф-2_2025-Q3.xlsx new file mode 100644 index 0000000..86b52c6 Binary files /dev/null and b/input/demo-periods/Ф-2_2025-Q3.xlsx differ diff --git a/input/demo-periods/Ф-2_2025-Q4.xlsx b/input/demo-periods/Ф-2_2025-Q4.xlsx new file mode 100644 index 0000000..4e6bb37 Binary files /dev/null and b/input/demo-periods/Ф-2_2025-Q4.xlsx differ diff --git a/input/demo-periods/Ф-2_2026-Q1.xlsx b/input/demo-periods/Ф-2_2026-Q1.xlsx new file mode 100644 index 0000000..4469acc Binary files /dev/null and b/input/demo-periods/Ф-2_2026-Q1.xlsx differ diff --git a/input/demo-periods/Ф-2_2026-Q2.xlsx b/input/demo-periods/Ф-2_2026-Q2.xlsx new file mode 100644 index 0000000..c9cc0b7 Binary files /dev/null and b/input/demo-periods/Ф-2_2026-Q2.xlsx differ diff --git a/input/demo-periods/Ф-3_2022.xlsx b/input/demo-periods/Ф-3_2022.xlsx new file mode 100644 index 0000000..51d935b Binary files /dev/null and b/input/demo-periods/Ф-3_2022.xlsx differ diff --git a/input/demo-periods/Ф-3_2023.xlsx b/input/demo-periods/Ф-3_2023.xlsx new file mode 100644 index 0000000..9e96cfc Binary files /dev/null and b/input/demo-periods/Ф-3_2023.xlsx differ diff --git a/input/demo-periods/Ф-3_2024.xlsx b/input/demo-periods/Ф-3_2024.xlsx new file mode 100644 index 0000000..be590ea Binary files /dev/null and b/input/demo-periods/Ф-3_2024.xlsx differ diff --git a/input/demo-periods/Ф-3_2025.xlsx b/input/demo-periods/Ф-3_2025.xlsx new file mode 100644 index 0000000..46bc5f8 Binary files /dev/null and b/input/demo-periods/Ф-3_2025.xlsx differ diff --git a/input/demo-periods/Ф-4_2024-H2.xlsx b/input/demo-periods/Ф-4_2024-H2.xlsx new file mode 100644 index 0000000..e3faab5 Binary files /dev/null and b/input/demo-periods/Ф-4_2024-H2.xlsx differ diff --git a/input/demo-periods/Ф-4_2025-H1.xlsx b/input/demo-periods/Ф-4_2025-H1.xlsx new file mode 100644 index 0000000..e5ec366 Binary files /dev/null and b/input/demo-periods/Ф-4_2025-H1.xlsx differ diff --git a/input/demo-periods/Ф-4_2025-H2.xlsx b/input/demo-periods/Ф-4_2025-H2.xlsx new file mode 100644 index 0000000..32ac433 Binary files /dev/null and b/input/demo-periods/Ф-4_2025-H2.xlsx differ diff --git a/input/demo-periods/Ф-4_2026-H1.xlsx b/input/demo-periods/Ф-4_2026-H1.xlsx new file mode 100644 index 0000000..0004938 Binary files /dev/null and b/input/demo-periods/Ф-4_2026-H1.xlsx differ diff --git a/input/demo-periods/Ф-5_2022.xlsx b/input/demo-periods/Ф-5_2022.xlsx new file mode 100644 index 0000000..5095e08 Binary files /dev/null and b/input/demo-periods/Ф-5_2022.xlsx differ diff --git a/input/demo-periods/Ф-5_2023.xlsx b/input/demo-periods/Ф-5_2023.xlsx new file mode 100644 index 0000000..4f0f708 Binary files /dev/null and b/input/demo-periods/Ф-5_2023.xlsx differ diff --git a/input/demo-periods/Ф-5_2024.xlsx b/input/demo-periods/Ф-5_2024.xlsx new file mode 100644 index 0000000..b07e6ea Binary files /dev/null and b/input/demo-periods/Ф-5_2024.xlsx differ diff --git a/input/demo-periods/Ф-5_2025.xlsx b/input/demo-periods/Ф-5_2025.xlsx new file mode 100644 index 0000000..a432213 Binary files /dev/null and b/input/demo-periods/Ф-5_2025.xlsx differ diff --git a/input/demo-periods/Ф-6_2022.xlsx b/input/demo-periods/Ф-6_2022.xlsx new file mode 100644 index 0000000..dfb6910 Binary files /dev/null and b/input/demo-periods/Ф-6_2022.xlsx differ diff --git a/input/demo-periods/Ф-6_2023.xlsx b/input/demo-periods/Ф-6_2023.xlsx new file mode 100644 index 0000000..b204cfd Binary files /dev/null and b/input/demo-periods/Ф-6_2023.xlsx differ diff --git a/input/demo-periods/Ф-6_2024.xlsx b/input/demo-periods/Ф-6_2024.xlsx new file mode 100644 index 0000000..96b2c3e Binary files /dev/null and b/input/demo-periods/Ф-6_2024.xlsx differ diff --git a/input/demo-periods/Ф-6_2025.xlsx b/input/demo-periods/Ф-6_2025.xlsx new file mode 100644 index 0000000..60f7822 Binary files /dev/null and b/input/demo-periods/Ф-6_2025.xlsx differ diff --git a/input/ПЕРИОДЫ-ДЛЯ-ДЕМО.md b/input/ПЕРИОДЫ-ДЛЯ-ДЕМО.md new file mode 100644 index 0000000..6c1199e --- /dev/null +++ b/input/ПЕРИОДЫ-ДЛЯ-ДЕМО.md @@ -0,0 +1,32 @@ +# Периоды загрузки демонстрационных форм + +Отчётный период передаётся в API отдельно от Excel-файла. Полный набор находится в каталоге `demo-periods/`. + +| Файл | Параметры периода | Отображаемый период | +|---|---|---| +| `Ф-1_2025-09.xlsx` | `report_year=2025`, `report_month=9` | Сентябрь 2025 | +| `Ф-1_2025-12.xlsx` | `report_year=2025`, `report_month=12` | Декабрь 2025 | +| `Ф-1_2026-03.xlsx` | `report_year=2026`, `report_month=3` | Март 2026 | +| `Ф-1_2026-06.xlsx` | `report_year=2026`, `report_month=6` | Июнь 2026 | +| `Ф-2_2025-Q3.xlsx` | `report_year=2025`, `report_quarter=3` | III квартал 2025 | +| `Ф-2_2025-Q4.xlsx` | `report_year=2025`, `report_quarter=4` | IV квартал 2025 | +| `Ф-2_2026-Q1.xlsx` | `report_year=2026`, `report_quarter=1` | I квартал 2026 | +| `Ф-2_2026-Q2.xlsx` | `report_year=2026`, `report_quarter=2` | II квартал 2026 | +| `Ф-3_2022.xlsx` | `report_year=2022` | 2022 год | +| `Ф-3_2023.xlsx` | `report_year=2023` | 2023 год | +| `Ф-3_2024.xlsx` | `report_year=2024` | 2024 год | +| `Ф-3_2025.xlsx` | `report_year=2025` | 2025 год | +| `Ф-4_2024-H2.xlsx` | `report_year=2024`, `report_half_year=2` | II полугодие 2024 | +| `Ф-4_2025-H1.xlsx` | `report_year=2025`, `report_half_year=1` | I полугодие 2025 | +| `Ф-4_2025-H2.xlsx` | `report_year=2025`, `report_half_year=2` | II полугодие 2025 | +| `Ф-4_2026-H1.xlsx` | `report_year=2026`, `report_half_year=1` | I полугодие 2026 | +| `Ф-5_2022.xlsx` | `report_year=2022` | 2022 год | +| `Ф-5_2023.xlsx` | `report_year=2023` | 2023 год | +| `Ф-5_2024.xlsx` | `report_year=2024` | 2024 год | +| `Ф-5_2025.xlsx` | `report_year=2025` | 2025 год | +| `Ф-6_2022.xlsx` | `report_year=2022` | 2022 год | +| `Ф-6_2023.xlsx` | `report_year=2023` | 2023 год | +| `Ф-6_2024.xlsx` | `report_year=2024` | 2024 год | +| `Ф-6_2025.xlsx` | `report_year=2025` | 2025 год | + +Каждый файл содержит только организации `Тестовая компания 1` — `Тестовая компания 20`. ИНН, ОГРН и ОКПО совпадают с данными команды Mostovik `create_test_companies`. Показатели плавно растут от ранних периодов к поздним, чтобы графики и сравнения на фронте показывали заметную динамику. diff --git a/input/Ф-1.xlsx b/input/Ф-1.xlsx index 9950bb5..6087128 100644 Binary files a/input/Ф-1.xlsx and b/input/Ф-1.xlsx differ diff --git a/input/Ф-2.xlsx b/input/Ф-2.xlsx index d494c8e..fee2bf4 100644 Binary files a/input/Ф-2.xlsx and b/input/Ф-2.xlsx differ diff --git a/input/Ф-3.xlsx b/input/Ф-3.xlsx index 8095582..5dd0e49 100644 Binary files a/input/Ф-3.xlsx and b/input/Ф-3.xlsx differ diff --git a/input/Ф-4.xlsx b/input/Ф-4.xlsx index f3a523a..604f6d6 100644 Binary files a/input/Ф-4.xlsx and b/input/Ф-4.xlsx differ diff --git a/input/Ф-5.xlsx b/input/Ф-5.xlsx index bb1ae39..8f497b3 100644 Binary files a/input/Ф-5.xlsx and b/input/Ф-5.xlsx differ diff --git a/input/Ф-6.xlsx b/input/Ф-6.xlsx index 46df81d..a42d043 100644 Binary files a/input/Ф-6.xlsx and b/input/Ф-6.xlsx differ diff --git a/src/apps/core/migrations/0002_report_upload.py b/src/apps/core/migrations/0002_report_upload.py new file mode 100644 index 0000000..4a449dd --- /dev/null +++ b/src/apps/core/migrations/0002_report_upload.py @@ -0,0 +1,110 @@ +import uuid + +import apps.core.models +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0001_background_job"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ReportUpload", + fields=[ + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + db_index=True, + help_text="Дата и время создания записи", + verbose_name="создано", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="Дата и время последнего обновления", + verbose_name="обновлено", + ), + ), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "form", + models.CharField( + choices=[ + ("f1", "Ф-1"), + ("f2", "Ф-2"), + ("f3", "Ф-3"), + ("f4", "Ф-4"), + ("f5", "Ф-5"), + ("f6", "Ф-6"), + ], + db_index=True, + max_length=2, + ), + ), + ( + "load_batch", + models.PositiveIntegerField(blank=True, db_index=True, null=True), + ), + ( + "original_file", + models.FileField( + max_length=500, + upload_to=apps.core.models.report_upload_path, + ), + ), + ("file_name", models.CharField(max_length=255)), + ("content_type", models.CharField(max_length=127)), + ("file_size", models.PositiveBigIntegerField()), + ("file_hash", models.CharField(db_index=True, max_length=64)), + ( + "status", + models.CharField( + choices=[ + ("queued", "Ожидает обработки"), + ("processing", "Обрабатывается"), + ("success", "Обработан"), + ("failed", "Ошибка"), + ], + db_index=True, + default="processing", + max_length=16, + ), + ), + ( + "uploaded_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="report_uploads", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={"ordering": ["-created_at"]}, + ), + migrations.AddConstraint( + model_name="reportupload", + constraint=models.UniqueConstraint( + condition=models.Q(load_batch__isnull=False), + fields=("form", "load_batch"), + name="unique_report_upload_form_batch", + ), + ), + ] diff --git a/src/apps/core/migrations/0003_report_upload_history.py b/src/apps/core/migrations/0003_report_upload_history.py new file mode 100644 index 0000000..6a753c8 --- /dev/null +++ b/src/apps/core/migrations/0003_report_upload_history.py @@ -0,0 +1,98 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("core", "0002_report_upload")] + + operations = [ + migrations.AddField( + model_name="reportupload", + name="report_year", + field=models.PositiveSmallIntegerField( + blank=True, db_index=True, null=True + ), + ), + migrations.AddField( + model_name="reportupload", + name="report_month", + field=models.PositiveSmallIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name="reportupload", + name="report_quarter", + field=models.PositiveSmallIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name="reportupload", + name="report_half_year", + field=models.PositiveSmallIntegerField(blank=True, null=True), + ), + migrations.AddField( + model_name="reportupload", + name="corporation_scope", + field=models.CharField(blank=True, db_index=True, max_length=32), + ), + migrations.AddField( + model_name="reportupload", + name="status_message", + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name="reportupload", + name="job_id", + field=models.CharField( + blank=True, db_index=True, max_length=255, null=True + ), + ), + migrations.AddField( + model_name="reportupload", + name="records_count", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="reportupload", + name="loaded_count", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="reportupload", + name="skipped_count", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="reportupload", + name="errors_count", + field=models.PositiveIntegerField(default=0), + ), + migrations.AddField( + model_name="reportupload", + name="validation_errors", + field=models.JSONField(blank=True, default=list), + ), + migrations.AddField( + model_name="reportupload", + name="started_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="reportupload", + name="completed_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AlterField( + model_name="reportupload", + name="status", + field=models.CharField( + choices=[ + ("queued", "Ожидает обработки"), + ("processing", "Обрабатывается"), + ("success", "Обработан"), + ("partial", "Обработан частично"), + ("failed", "Ошибка"), + ], + db_index=True, + default="processing", + max_length=16, + ), + ), + ] diff --git a/src/apps/core/models.py b/src/apps/core/models.py index 9b4cb7c..882f6ef 100644 --- a/src/apps/core/models.py +++ b/src/apps/core/models.py @@ -8,14 +8,85 @@ Background Job Tracking - отслеживание статуса Celery зад """ import uuid +from pathlib import Path from typing import Any from apps.core.mixins import TimestampMixin +from django.conf import settings from django.db import models from django.utils import timezone from django.utils.translation import gettext_lazy as _ +def report_upload_path(instance, filename: str) -> str: + """Build a storage key without trusting client-supplied path components.""" + safe_name = Path(filename).name or "report.xlsx" + return f"report_uploads/{instance.form}/{instance.id}/{safe_name}" + + +class ReportUploadStatus(models.TextChoices): + QUEUED = "queued", _("Ожидает обработки") + PROCESSING = "processing", _("Обрабатывается") + SUCCESS = "success", _("Обработан") + PARTIAL = "partial", _("Обработан частично") + FAILED = "failed", _("Ошибка") + + +class ReportUpload(TimestampMixin, models.Model): + """Persistent original file for one F-1 ... F-6 report upload.""" + + FORM_CHOICES = tuple((f"f{number}", f"Ф-{number}") for number in range(1, 7)) + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + form = models.CharField(max_length=2, choices=FORM_CHOICES, db_index=True) + load_batch = models.PositiveIntegerField(null=True, blank=True, db_index=True) + report_year = models.PositiveSmallIntegerField(null=True, blank=True, db_index=True) + report_month = models.PositiveSmallIntegerField(null=True, blank=True) + report_quarter = models.PositiveSmallIntegerField(null=True, blank=True) + report_half_year = models.PositiveSmallIntegerField(null=True, blank=True) + corporation_scope = models.CharField(max_length=32, blank=True, db_index=True) + original_file = models.FileField(upload_to=report_upload_path, max_length=500) + file_name = models.CharField(max_length=255) + content_type = models.CharField(max_length=127) + file_size = models.PositiveBigIntegerField() + file_hash = models.CharField(max_length=64, db_index=True) + status = models.CharField( + max_length=16, + choices=ReportUploadStatus.choices, + default=ReportUploadStatus.PROCESSING, + db_index=True, + ) + status_message = models.TextField(blank=True) + job_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) + records_count = models.PositiveIntegerField(default=0) + loaded_count = models.PositiveIntegerField(default=0) + skipped_count = models.PositiveIntegerField(default=0) + errors_count = models.PositiveIntegerField(default=0) + validation_errors = models.JSONField(default=list, blank=True) + started_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + uploaded_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="report_uploads", + ) + + class Meta: + ordering = ["-created_at"] + constraints = [ + models.UniqueConstraint( + fields=["form", "load_batch"], + condition=models.Q(load_batch__isnull=False), + name="unique_report_upload_form_batch", + ) + ] + + def __str__(self) -> str: + return f"{self.form}:{self.load_batch or self.id}" + + class JobStatus(models.TextChoices): """Статусы фоновых задач.""" diff --git a/src/apps/core/report_upload_history.py b/src/apps/core/report_upload_history.py new file mode 100644 index 0000000..cf23602 --- /dev/null +++ b/src/apps/core/report_upload_history.py @@ -0,0 +1,271 @@ +"""Read projection for report upload history.""" + +from __future__ import annotations + +from collections import defaultdict +from uuid import NAMESPACE_URL, uuid5 + +from apps.core.models import ReportUpload, ReportUploadStatus +from apps.core.report_uploads import ALLOWED_FORMS, FORM_RECORD_MODELS +from apps.core.upload_contracts import ( + report_annual_display, + report_half_year_display, + report_month_display, + report_quarter_display, +) +from apps.organization.models import CorporationScope, Organization +from apps.organization.scope_utils import SCOPE_LABELS +from django.apps import apps +from django.db.models import Count, Max, Q +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import serializers +from rest_framework.pagination import PageNumberPagination +from rest_framework.permissions import IsAuthenticated +from rest_framework.views import APIView + + +class ReportUploadHistoryQuerySerializer(serializers.Serializer): + form = serializers.ChoiceField(choices=sorted(ALLOWED_FORMS), required=False) + corporation_scope = serializers.ChoiceField( + choices=CorporationScope.values, required=False + ) + organization_id = serializers.UUIDField(required=False) + organization_inn = serializers.CharField(required=False) + search = serializers.CharField(required=False, allow_blank=True) + report_year = serializers.IntegerField(min_value=2000, required=False) + report_month = serializers.IntegerField(min_value=1, max_value=12, required=False) + report_quarter = serializers.IntegerField(min_value=1, max_value=4, required=False) + report_half_year = serializers.IntegerField( + min_value=1, max_value=2, required=False + ) + status = serializers.ChoiceField(choices=ReportUploadStatus.values, required=False) + upload_id = serializers.UUIDField(required=False) + load_batch = serializers.IntegerField(min_value=1, required=False) + created_from = serializers.DateTimeField(required=False) + created_to = serializers.DateTimeField(required=False) + ordering = serializers.ChoiceField( + choices=( + "created_at", + "-created_at", + "status", + "-status", + "form", + "-form", + "report_year", + "-report_year", + "load_batch", + "-load_batch", + ), + default="-created_at", + ) + page = serializers.IntegerField(min_value=1, required=False) + page_size = serializers.IntegerField(min_value=1, max_value=100, required=False) + + def validate(self, attrs): + form = attrs.get("form") + if attrs.get("report_month") is not None and form not in {None, "f1"}: + raise serializers.ValidationError( + {"report_month": "Параметр доступен только для формы Ф-1"} + ) + if attrs.get("report_half_year") is not None and form not in {None, "f4"}: + raise serializers.ValidationError( + {"report_half_year": "Параметр доступен только для формы Ф-4"} + ) + return attrs + + +class ReportUploadHistoryPagination(PageNumberPagination): + page_size = 20 + page_size_query_param = "page_size" + max_page_size = 100 + + +def _period_display(upload: ReportUpload) -> str | None: + if upload.report_year is None: + return None + if upload.report_month is not None: + return report_month_display(upload.report_year, upload.report_month) + if upload.report_half_year is not None: + return report_half_year_display(upload.report_year, upload.report_half_year) + if upload.report_quarter is not None: + return report_quarter_display(upload.report_year, upload.report_quarter) + return report_annual_display(upload.report_year) + + +def _uploaded_by(upload: ReportUpload) -> dict[str, str] | None: + user = upload.uploaded_by + if user is None: + return None + profile = getattr(user, "profile", None) + name = profile.full_name if profile is not None else user.username + return {"id": str(user.id), "name": name} + + +def _record_projection(uploads: list[ReportUpload]): + projection: dict[tuple[str, int], list[dict[str, object]]] = defaultdict(list) + organization_ids: set[object] = set() + + for form in ALLOWED_FORMS: + batches = { + upload.load_batch + for upload in uploads + if upload.form == form and upload.load_batch is not None + } + if not batches: + continue + app_label, model_name = FORM_RECORD_MODELS[form] + record_model = apps.get_model(app_label, model_name) + records = list( + record_model.objects.filter(load_batch__in=batches) + .values("load_batch", "organization_id") + .annotate( + records_count=Count("id"), + active_records_count=Count("id", filter=Q(is_active_version=True)), + superseded_by_batch=Max("superseded_by_batch"), + ) + ) + for record in records: + organization_ids.add(record["organization_id"]) + projection[(form, record["load_batch"])].append(record) + + return projection, Organization.objects.in_bulk(organization_ids) + + +def _download_available(upload: ReportUpload) -> bool: + return bool( + upload.load_batch is not None + and upload.original_file + and upload.original_file.storage.exists(upload.original_file.name) + ) + + +def _row( + upload: ReportUpload, + record: dict[str, object] | None, + organizations: dict[object, Organization], +) -> dict[str, object]: + organization = organizations.get(record["organization_id"]) if record else None + scopes = organization.get_corporation_scopes() if organization else [] + corporation_scope = upload.corporation_scope or (scopes[0] if scopes else None) + available = _download_available(upload) + row_id = ( + uuid5(NAMESPACE_URL, f"{upload.id}:{organization.id}") + if organization is not None + else upload.id + ) + records_count = int(record["records_count"]) if record else upload.records_count + return { + "id": str(row_id), + "upload_id": str(upload.id), + "form": upload.form, + "form_display": upload.get_form_display(), + "load_batch": upload.load_batch, + "file_name": upload.file_name, + "file_size": upload.file_size, + "content_type": upload.content_type, + "corporation_scope": corporation_scope, + "corporation_scope_label": SCOPE_LABELS.get(corporation_scope), + "organization_id": str(organization.id) if organization else None, + "organization_name": organization.name if organization else None, + "organization_inn": organization.inn if organization else None, + "report_year": upload.report_year, + "report_month": upload.report_month, + "report_quarter": upload.report_quarter, + "report_half_year": upload.report_half_year, + "report_period_display": _period_display(upload), + "status": upload.status, + "status_message": upload.status_message or None, + "job_id": upload.job_id, + "records_count": records_count, + "loaded_count": records_count if record else upload.loaded_count, + "skipped_count": upload.skipped_count, + "errors_count": upload.errors_count, + "is_active_version": (bool(record["active_records_count"]) if record else None), + "superseded_by_batch": record["superseded_by_batch"] if record else None, + "created_at": upload.created_at, + "started_at": upload.started_at, + "completed_at": upload.completed_at, + "updated_at": upload.updated_at, + "uploaded_by": _uploaded_by(upload), + "download_available": available, + "download_url": ( + f"/api/v1/forms/{upload.form}/uploads/{upload.load_batch}/download/" + if available + else None + ), + "validation_errors": upload.validation_errors, + } + + +def _matches(row: dict[str, object], filters: dict[str, object]) -> bool: + direct_fields = ( + "corporation_scope", + "organization_id", + "organization_inn", + "report_year", + "report_month", + "report_quarter", + "report_half_year", + ) + if any( + field in filters and str(row.get(field)) != str(filters[field]) + for field in direct_fields + ): + return False + search = str(filters.get("search") or "").casefold() + if search and not any( + search in str(row.get(field) or "").casefold() + for field in ("organization_name", "organization_inn", "file_name") + ): + return False + return True + + +class ReportUploadHistoryView(APIView): + permission_classes = [IsAuthenticated] + pagination_class = ReportUploadHistoryPagination + + @swagger_auto_schema( + tags=["Формы отчетности"], + operation_summary="История загрузок форм отчетности", + query_serializer=ReportUploadHistoryQuerySerializer, + responses={200: openapi.Response("Пагинированная история загрузок")}, + ) + def get(self, request): + serializer = ReportUploadHistoryQuerySerializer(data=request.query_params) + serializer.is_valid(raise_exception=True) + filters = serializer.validated_data + + queryset = ReportUpload.objects.select_related("uploaded_by__profile") + if not request.user.is_staff: + queryset = queryset.filter(uploaded_by=request.user) + for field in ("form", "status", "load_batch"): + if field in filters: + queryset = queryset.filter(**{field: filters[field]}) + if "upload_id" in filters: + queryset = queryset.filter(pk=filters["upload_id"]) + if "created_from" in filters: + queryset = queryset.filter(created_at__gte=filters["created_from"]) + if "created_to" in filters: + queryset = queryset.filter(created_at__lte=filters["created_to"]) + + uploads = list(queryset) + projection, organizations = _record_projection(uploads) + rows = [ + _row(upload, record, organizations) + for upload in uploads + for record in (projection.get((upload.form, upload.load_batch)) or [None]) + ] + rows = [row for row in rows if _matches(row, filters)] + + ordering = filters["ordering"] + reverse = ordering.startswith("-") + field = ordering.lstrip("-") + rows.sort( + key=lambda row: (row.get(field) is not None, row.get(field), row["id"]), + reverse=reverse, + ) + paginator = self.pagination_class() + page = paginator.paginate_queryset(rows, request, view=self) + return paginator.get_paginated_response(page) diff --git a/src/apps/core/report_uploads.py b/src/apps/core/report_uploads.py new file mode 100644 index 0000000..82bca9e --- /dev/null +++ b/src/apps/core/report_uploads.py @@ -0,0 +1,216 @@ +"""Persistence and download contract for original reporting files.""" + +from __future__ import annotations + +import hashlib +import mimetypes +from pathlib import Path +from typing import Any +from uuid import UUID + +from apps.core.models import ReportUpload, ReportUploadStatus +from apps.core.response import api_error_response +from django.apps import apps +from django.core.files.base import ContentFile +from django.http import FileResponse +from django.utils import timezone +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework.permissions import IsAuthenticated +from rest_framework.views import APIView + +ALLOWED_FORMS = frozenset(f"f{number}" for number in range(1, 7)) +FORM_RECORD_MODELS = { + f"f{number}": (f"form_{number}", f"FormF{number}Record") for number in range(1, 7) +} +EXCEL_CONTENT_TYPES = { + ".xls": "application/vnd.ms-excel", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", +} + + +class ReportUploadService: + """Own lifecycle transitions for a persisted report upload.""" + + @staticmethod + def create( + *, + uploaded_file, + form: str, + user=None, + report_year: int | None = None, + report_month: int | None = None, + report_quarter: int | None = None, + report_half_year: int | None = None, + corporation_scope: str = "", + ) -> ReportUpload: + if form not in ALLOWED_FORMS: + raise ValueError(f"Unsupported report form: {form}") + + file_name = Path(str(uploaded_file.name)).name + content = uploaded_file.read() + uploaded_file.seek(0) + suffix = Path(file_name).suffix.lower() + content_type = EXCEL_CONTENT_TYPES.get(suffix) or ( + mimetypes.guess_type(file_name)[0] or "application/octet-stream" + ) + report_upload = ReportUpload( + form=form, + file_name=file_name, + content_type=content_type, + file_size=len(content), + file_hash=hashlib.sha256(content).hexdigest(), + status=ReportUploadStatus.PROCESSING, + report_year=report_year, + report_month=report_month, + report_quarter=report_quarter, + report_half_year=report_half_year, + corporation_scope=corporation_scope, + started_at=timezone.now(), + uploaded_by=user if getattr(user, "is_authenticated", False) else None, + ) + report_upload.original_file.save( + file_name, + ContentFile(content), + save=False, + ) + report_upload.save() + return report_upload + + @staticmethod + def finalize(upload_id: UUID | str, result: Any) -> ReportUpload: + batch_id = ( + result.get("batch_id") + if isinstance(result, dict) + else getattr(result, "batch_id", None) + ) + if batch_id is None: + raise ValueError("Parser result does not contain batch_id") + + report_upload = ReportUpload.objects.get(pk=upload_id) + app_label, model_name = FORM_RECORD_MODELS[report_upload.form] + record_model = apps.get_model(app_label, model_name) + records_count = record_model.objects.filter(load_batch=batch_id).count() + report_upload.load_batch = int(batch_id) + report_upload.status = ReportUploadStatus.SUCCESS + report_upload.status_message = "" + report_upload.records_count = records_count + report_upload.loaded_count = records_count + report_upload.completed_at = timezone.now() + report_upload.save( + update_fields=[ + "load_batch", + "status", + "status_message", + "records_count", + "loaded_count", + "completed_at", + "updated_at", + ] + ) + return report_upload + + @staticmethod + def mark_failed( + upload_id: UUID | str, + *, + message: str = "Не удалось обработать файл", + validation_errors: list[dict[str, object]] | None = None, + ) -> None: + ReportUpload.objects.filter(pk=upload_id).update( + status=ReportUploadStatus.FAILED, + status_message=message, + validation_errors=validation_errors or [], + errors_count=len(validation_errors or []), + completed_at=timezone.now(), + ) + + @staticmethod + def mark_queued(upload_id: UUID | str, *, job_id: str | None = None) -> None: + ReportUpload.objects.filter(pk=upload_id).update( + status=ReportUploadStatus.QUEUED, + status_message="Файл принят и ожидает обработки", + job_id=job_id, + ) + + @staticmethod + def mark_processing(upload_id: UUID | str) -> None: + ReportUpload.objects.filter(pk=upload_id).update( + status=ReportUploadStatus.PROCESSING, + status_message="Файл принят и обрабатывается", + started_at=timezone.now(), + ) + + +class ReportUploadDownloadView(APIView): + """Stream the unchanged original Excel file for a processed load batch.""" + + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + tags=["Формы отчетности"], + operation_summary="Скачать исходный файл загрузки отчетности", + responses={ + 200: openapi.Response( + "Исходный .xls или .xlsx файл", + schema=openapi.Schema(type=openapi.TYPE_STRING, format="binary"), + ), + 401: "Требуется JWT-аутентификация", + 403: "Нет доступа к загрузке", + 404: "Загрузка не найдена", + 410: "Исходный файл был удален", + }, + ) + def get(self, request, form: str, load_batch: int): + if form not in ALLOWED_FORMS: + return api_error_response( + [{"code": "upload_not_found", "message": "Загрузка не найдена."}], + status_code=status.HTTP_404_NOT_FOUND, + ) + + report_upload = ReportUpload.objects.filter( + form=form, + load_batch=load_batch, + ).first() + if report_upload is None: + return api_error_response( + [{"code": "upload_not_found", "message": "Загрузка не найдена."}], + status_code=status.HTTP_404_NOT_FOUND, + ) + if ( + report_upload.uploaded_by_id != request.user.id + and not request.user.is_staff + ): + return api_error_response( + [ + { + "code": "upload_forbidden", + "message": "Нет доступа к исходному файлу загрузки.", + } + ], + status_code=status.HTTP_403_FORBIDDEN, + ) + if ( + not report_upload.original_file + or not report_upload.original_file.storage.exists( + report_upload.original_file.name + ) + ): + return api_error_response( + [ + { + "code": "source_file_gone", + "message": "Исходный файл загрузки удален.", + } + ], + status_code=status.HTTP_410_GONE, + ) + + file_handle = report_upload.original_file.open("rb") + return FileResponse( + file_handle, + as_attachment=True, + filename=Path(report_upload.file_name).name, + content_type=report_upload.content_type, + ) diff --git a/src/apps/core/upload_contracts.py b/src/apps/core/upload_contracts.py index 3a9a1ec..f004ced 100644 --- a/src/apps/core/upload_contracts.py +++ b/src/apps/core/upload_contracts.py @@ -79,10 +79,11 @@ def build_upload_success_payload( result: dict[str, Any] | None = None, job_id: str | None = None, created_at: datetime | None = None, + upload_id: str | None = None, ) -> dict[str, Any]: created = created_at or timezone.now() payload: dict[str, Any] = { - "upload_id": str(uuid4()), + "upload_id": upload_id or str(uuid4()), "form": form, "report_year": report_year, "status": status, diff --git a/src/apps/exchange/serializers.py b/src/apps/exchange/serializers.py index c19b4e2..202c0f1 100644 --- a/src/apps/exchange/serializers.py +++ b/src/apps/exchange/serializers.py @@ -1,5 +1,6 @@ """Serializers for exchange package upload endpoints.""" +from apps.exchange.models import ExchangePackageImport from rest_framework import serializers @@ -16,3 +17,21 @@ class ExchangePackageUploadSerializer(serializers.Serializer): "Поддерживаются только архивы .zip или контейнеры .bin." ) return value + + +class ExchangePackageImportSerializer(serializers.ModelSerializer): + """Frontend-facing metadata for an imported exchange package.""" + + class Meta: + model = ExchangePackageImport + fields = ( + "id", + "package_id", + "package_name", + "source_system", + "schema_version", + "delivery_channel", + "status", + "created_at", + ) + read_only_fields = fields diff --git a/src/apps/exchange/urls.py b/src/apps/exchange/urls.py index e37fad3..0e9ac33 100644 --- a/src/apps/exchange/urls.py +++ b/src/apps/exchange/urls.py @@ -1,11 +1,14 @@ """URL routes for exchange uploads.""" -from apps.exchange.views import ExchangePackageUploadView +from apps.exchange.views import ExchangePackageLatestView, ExchangePackageUploadView from django.urls import path app_name = "exchange" urlpatterns = [ + path( + "packages/latest/", ExchangePackageLatestView.as_view(), name="package-latest" + ), path( "packages/upload/", ExchangePackageUploadView.as_view(), name="package-upload" ), diff --git a/src/apps/exchange/views.py b/src/apps/exchange/views.py index e7c1c86..c89a5cf 100644 --- a/src/apps/exchange/views.py +++ b/src/apps/exchange/views.py @@ -2,8 +2,12 @@ from __future__ import annotations -from apps.exchange.models import ExchangeDeliveryChannel -from apps.exchange.serializers import ExchangePackageUploadSerializer +from apps.core.response import api_response +from apps.exchange.models import ExchangeDeliveryChannel, ExchangePackageImport +from apps.exchange.serializers import ( + ExchangePackageImportSerializer, + ExchangePackageUploadSerializer, +) from apps.exchange.services import ExchangeImportError, ExchangePackageImportService from django.conf import settings from django.utils.crypto import constant_time_compare @@ -11,13 +15,38 @@ from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.parsers import MultiPartParser -from rest_framework.permissions import AllowAny +from rest_framework.permissions import AllowAny, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView EXCHANGE_TAG = "Обмен данными" +class ExchangePackageLatestView(APIView): + """Return metadata for the most recent package import attempt.""" + + permission_classes = [IsAuthenticated] + + @swagger_auto_schema( + tags=[EXCHANGE_TAG], + operation_summary="Получить последнюю загрузку пакета обмена", + responses={ + 200: ExchangePackageImportSerializer(), + 401: "Требуется JWT-аутентификация", + }, + ) + def get(self, request): + package_import = ExchangePackageImport.objects.order_by( + "-created_at", "-id" + ).first() + data = ( + ExchangePackageImportSerializer(package_import).data + if package_import is not None + else None + ) + return api_response(data) + + class ExchangePackageUploadView(APIView): """Accept encrypted exchange package uploads for dev integration.""" diff --git a/src/apps/form_1/api.py b/src/apps/form_1/api.py index a4cd4b5..fde890d 100644 --- a/src/apps/form_1/api.py +++ b/src/apps/form_1/api.py @@ -8,6 +8,7 @@ API для формы Ф-1. import logging +from apps.core.report_uploads import ReportUploadService from apps.core.response import api_response from apps.core.upload_contracts import ( build_upload_error_response, @@ -140,6 +141,13 @@ class FormF1UploadView(APIView): file = serializer.validated_data["file"] report_year = serializer.validated_data["report_year"] report_month = serializer.validated_data["report_month"] + report_upload = ReportUploadService.create( + uploaded_file=file, + form="f1", + user=request.user, + report_year=report_year, + report_month=report_month, + ) # Определяем размер файла для выбора режима обработки file_size = file.size @@ -152,6 +160,7 @@ class FormF1UploadView(APIView): report_year=report_year, report_month=report_month, ) + ReportUploadService.finalize(report_upload.id, result) result_serializer = ParseResultSerializer(result) return Response( build_upload_success_payload( @@ -160,10 +169,12 @@ class FormF1UploadView(APIView): report_month=report_month, status="done", result=result_serializer.data, + upload_id=str(report_upload.id), ), status=status.HTTP_200_OK, ) except Exception as e: + ReportUploadService.mark_failed(report_upload.id) logger.exception("Ошибка обработки файла Ф-1") return build_upload_error_response( error_code="processing_error", @@ -182,7 +193,9 @@ class FormF1UploadView(APIView): user_id=request.user.id, report_year=report_year, report_month=report_month, + report_upload_id=str(report_upload.id), ) + ReportUploadService.mark_queued(report_upload.id, job_id=task.id) return Response( build_upload_success_payload( @@ -191,6 +204,7 @@ class FormF1UploadView(APIView): report_month=report_month, status="queued", job_id=task.id, + upload_id=str(report_upload.id), ), status=status.HTTP_202_ACCEPTED, ) diff --git a/src/apps/form_1/tasks.py b/src/apps/form_1/tasks.py index 03b775b..ecf1fef 100644 --- a/src/apps/form_1/tasks.py +++ b/src/apps/form_1/tasks.py @@ -8,6 +8,7 @@ Celery задачи для формы Ф-1. import logging from contextlib import suppress +from apps.core.report_uploads import ReportUploadService from apps.core.tasks import TrackedTask from apps.form_1.services import parse_form_f1_file from celery import shared_task @@ -25,6 +26,7 @@ def process_form_f1_file( report_year: int | None = None, report_month: int | None = None, report_quarter: int | None = None, + report_upload_id: str | None = None, ): """ Фоновая обработка файла формы Ф-1. @@ -40,6 +42,8 @@ def process_form_f1_file( job.update_progress(10, "Загрузка файла...") try: + if report_upload_id is not None: + ReportUploadService.mark_processing(report_upload_id) # Открываем файл из storage with default_storage.open(file_path, "rb") as f: job.update_progress(20, "Парсинг данных...") @@ -52,6 +56,8 @@ def process_form_f1_file( or (report_quarter * 3 if report_quarter is not None else None) or timezone.now().month, ) + if report_upload_id is not None: + ReportUploadService.finalize(report_upload_id, result) job.update_progress(90, "Завершение...") @@ -61,6 +67,8 @@ def process_form_f1_file( return result.to_dict() except Exception as e: + if report_upload_id is not None: + ReportUploadService.mark_failed(report_upload_id) logger.exception(f"Ошибка обработки файла Ф-1: {e}") # Пытаемся удалить файл при ошибке with suppress(Exception): diff --git a/src/apps/form_2/api.py b/src/apps/form_2/api.py index 6120006..f6c3783 100644 --- a/src/apps/form_2/api.py +++ b/src/apps/form_2/api.py @@ -8,6 +8,7 @@ API формы Ф-2. import logging +from apps.core.report_uploads import ReportUploadService from apps.core.upload_contracts import ( build_upload_error_response, build_upload_success_payload, @@ -65,6 +66,13 @@ class FormF2UploadView(APIView): file = serializer.validated_data["file"] report_year = serializer.validated_data["report_year"] report_quarter = serializer.validated_data.get("report_quarter") + report_upload = ReportUploadService.create( + uploaded_file=file, + form="f2", + user=request.user, + report_year=report_year, + report_quarter=report_quarter, + ) # Для больших файлов - фоновая обработка if file.size > BACKGROUND_THRESHOLD: @@ -74,7 +82,9 @@ class FormF2UploadView(APIView): file.name, report_year, report_quarter, + str(report_upload.id), ) + ReportUploadService.mark_queued(report_upload.id, job_id=task.id) return Response( build_upload_success_payload( @@ -83,6 +93,7 @@ class FormF2UploadView(APIView): report_quarter=report_quarter, status="queued", job_id=task.id, + upload_id=str(report_upload.id), ), status=status.HTTP_202_ACCEPTED, ) @@ -94,6 +105,7 @@ class FormF2UploadView(APIView): report_year=report_year, report_quarter=report_quarter, ) + ReportUploadService.finalize(report_upload.id, result) result_serializer = FormF2ParseResultSerializer(result) return Response( @@ -103,10 +115,12 @@ class FormF2UploadView(APIView): report_quarter=report_quarter, status="done", result=result_serializer.data, + upload_id=str(report_upload.id), ), status=status.HTTP_200_OK, ) except Exception as e: + ReportUploadService.mark_failed(report_upload.id) logger.exception("Ошибка обработки файла Ф-2") return build_upload_error_response( error_code="processing_error", diff --git a/src/apps/form_2/tasks.py b/src/apps/form_2/tasks.py index d292b1f..be4d54a 100644 --- a/src/apps/form_2/tasks.py +++ b/src/apps/form_2/tasks.py @@ -7,6 +7,7 @@ Celery задачи для формы Ф-2. import logging +from apps.core.report_uploads import ReportUploadService from apps.core.tasks import TrackedTask from apps.form_2.services import FormF2Parser from celery import shared_task @@ -21,6 +22,7 @@ def process_form_f2_file( file_name: str, report_year: int, report_quarter: int | None = None, + report_upload_id: str | None = None, ) -> dict: """ Фоновая обработка файла формы Ф-2. @@ -40,8 +42,17 @@ def process_form_f2_file( file_io = BytesIO(file_content) # Парсим файл + if report_upload_id is not None: + ReportUploadService.mark_processing(report_upload_id) parser = FormF2Parser(report_year=report_year, report_quarter=report_quarter) - result = parser.parse(file_io) + try: + result = parser.parse(file_io) + if report_upload_id is not None: + ReportUploadService.finalize(report_upload_id, result) + except Exception: + if report_upload_id is not None: + ReportUploadService.mark_failed(report_upload_id) + raise logger.info( f"Обработка Ф-2 завершена: загружено {result.loaded_count}, " diff --git a/src/apps/form_3/api.py b/src/apps/form_3/api.py index 84c10fd..0fcb4c5 100644 --- a/src/apps/form_3/api.py +++ b/src/apps/form_3/api.py @@ -8,6 +8,7 @@ API формы Ф-3. import logging +from apps.core.report_uploads import ReportUploadService from apps.core.upload_contracts import ( build_upload_error_response, build_upload_success_payload, @@ -64,6 +65,13 @@ class FormF3UploadView(APIView): file = serializer.validated_data["file"] report_year = serializer.validated_data["report_year"] report_quarter = serializer.validated_data.get("report_quarter") + report_upload = ReportUploadService.create( + uploaded_file=file, + form="f3", + user=request.user, + report_year=report_year, + report_quarter=report_quarter, + ) if file.size > BACKGROUND_THRESHOLD: file_content = file.read() @@ -72,7 +80,9 @@ class FormF3UploadView(APIView): file.name, report_year, report_quarter, + str(report_upload.id), ) + ReportUploadService.mark_queued(report_upload.id, job_id=task.id) return Response( build_upload_success_payload( @@ -80,6 +90,7 @@ class FormF3UploadView(APIView): report_year=report_year, status="queued", job_id=task.id, + upload_id=str(report_upload.id), ), status=status.HTTP_202_ACCEPTED, ) @@ -90,6 +101,7 @@ class FormF3UploadView(APIView): report_year=report_year, report_quarter=report_quarter, ) + ReportUploadService.finalize(report_upload.id, result) result_serializer = FormF3ParseResultSerializer(result) return Response( @@ -98,10 +110,12 @@ class FormF3UploadView(APIView): report_year=report_year, status="done", result=result_serializer.data, + upload_id=str(report_upload.id), ), status=status.HTTP_200_OK, ) except Exception as e: + ReportUploadService.mark_failed(report_upload.id) logger.exception("Ошибка обработки файла Ф-3") return build_upload_error_response( error_code="processing_error", diff --git a/src/apps/form_3/tasks.py b/src/apps/form_3/tasks.py index 1423678..fd47a69 100644 --- a/src/apps/form_3/tasks.py +++ b/src/apps/form_3/tasks.py @@ -7,6 +7,7 @@ Celery задачи для формы Ф-3. import logging +from apps.core.report_uploads import ReportUploadService from apps.core.tasks import TrackedTask from apps.form_3.services import FormF3Parser from celery import shared_task @@ -21,6 +22,7 @@ def process_form_f3_file( file_name: str, report_year: int, report_quarter: int | None = None, + report_upload_id: str | None = None, ) -> dict: """ Фоновая обработка файла формы Ф-3. @@ -37,8 +39,17 @@ def process_form_f3_file( logger.info(f"Начало обработки файла Ф-3: {file_name}") file_io = BytesIO(file_content) + if report_upload_id is not None: + ReportUploadService.mark_processing(report_upload_id) parser = FormF3Parser(report_year=report_year, report_quarter=report_quarter) - result = parser.parse(file_io) + try: + result = parser.parse(file_io) + if report_upload_id is not None: + ReportUploadService.finalize(report_upload_id, result) + except Exception: + if report_upload_id is not None: + ReportUploadService.mark_failed(report_upload_id) + raise logger.info( f"Обработка Ф-3 завершена: загружено {result.loaded_count}, " diff --git a/src/apps/form_4/api.py b/src/apps/form_4/api.py index d84e9ab..f6e745c 100644 --- a/src/apps/form_4/api.py +++ b/src/apps/form_4/api.py @@ -2,6 +2,7 @@ import logging +from apps.core.report_uploads import ReportUploadService from apps.core.upload_contracts import ( build_upload_error_response, build_upload_success_payload, @@ -49,6 +50,13 @@ class FormF4UploadView(APIView): file = serializer.validated_data["file"] report_year = serializer.validated_data["report_year"] report_half_year = serializer.validated_data["report_half_year"] + report_upload = ReportUploadService.create( + uploaded_file=file, + form="f4", + user=request.user, + report_year=report_year, + report_half_year=report_half_year, + ) if file.size > BACKGROUND_THRESHOLD: task = process_form_f4_file.delay( @@ -56,7 +64,10 @@ class FormF4UploadView(APIView): file.name, report_year, report_half_year, + None, + str(report_upload.id), ) + ReportUploadService.mark_queued(report_upload.id, job_id=task.id) return Response( build_upload_success_payload( form="f4", @@ -64,6 +75,7 @@ class FormF4UploadView(APIView): report_half_year=report_half_year, status="queued", job_id=task.id, + upload_id=str(report_upload.id), ), status=status.HTTP_202_ACCEPTED, ) @@ -74,6 +86,7 @@ class FormF4UploadView(APIView): report_year=report_year, report_half_year=report_half_year, ) + ReportUploadService.finalize(report_upload.id, result) return Response( build_upload_success_payload( form="f4", @@ -81,10 +94,12 @@ class FormF4UploadView(APIView): report_half_year=report_half_year, status="done", result=FormF4ParseResultSerializer(result).data, + upload_id=str(report_upload.id), ), status=status.HTTP_200_OK, ) except Exception as e: + ReportUploadService.mark_failed(report_upload.id) logger.exception("Ошибка обработки файла Ф-4") return build_upload_error_response( error_code="processing_error", diff --git a/src/apps/form_4/tasks.py b/src/apps/form_4/tasks.py index bb8e91d..dc4582c 100644 --- a/src/apps/form_4/tasks.py +++ b/src/apps/form_4/tasks.py @@ -3,6 +3,7 @@ import logging from io import BytesIO +from apps.core.report_uploads import ReportUploadService from apps.core.tasks import TrackedTask from apps.form_4.services import FormF4Parser from celery import shared_task @@ -18,13 +19,23 @@ def process_form_f4_file( report_year: int, report_half_year: int | None = None, report_quarter: int | None = None, + report_upload_id: str | None = None, ) -> dict: logger.info(f"Начало обработки файла Ф-4: {file_name}") + if report_upload_id is not None: + ReportUploadService.mark_processing(report_upload_id) parser = FormF4Parser( report_year=report_year, report_half_year=report_half_year or report_quarter, ) - result = parser.parse(BytesIO(file_content)) + try: + result = parser.parse(BytesIO(file_content)) + if report_upload_id is not None: + ReportUploadService.finalize(report_upload_id, result) + except Exception: + if report_upload_id is not None: + ReportUploadService.mark_failed(report_upload_id) + raise logger.info( f"Обработка Ф-4 завершена: {result.loaded_count} загружено, {result.skipped_count} пропущено" ) diff --git a/src/apps/form_5/api.py b/src/apps/form_5/api.py index a1d2c59..67a5ebd 100644 --- a/src/apps/form_5/api.py +++ b/src/apps/form_5/api.py @@ -2,6 +2,7 @@ import logging +from apps.core.report_uploads import ReportUploadService from apps.core.upload_contracts import ( build_upload_error_response, build_upload_success_payload, @@ -50,6 +51,13 @@ class FormF5UploadView(APIView): report_year = serializer.validated_data["report_year"] report_quarter = serializer.validated_data.get("report_quarter") + report_upload = ReportUploadService.create( + uploaded_file=file, + form="f5", + user=request.user, + report_year=report_year, + report_quarter=report_quarter, + ) if file.size > BACKGROUND_THRESHOLD: task = process_form_f5_file.delay( @@ -57,13 +65,16 @@ class FormF5UploadView(APIView): file.name, report_year, report_quarter, + str(report_upload.id), ) + ReportUploadService.mark_queued(report_upload.id, job_id=task.id) return Response( build_upload_success_payload( form="f5", report_year=report_year, status="queued", job_id=task.id, + upload_id=str(report_upload.id), ), status=status.HTTP_202_ACCEPTED, ) @@ -74,16 +85,19 @@ class FormF5UploadView(APIView): report_year=report_year, report_quarter=report_quarter, ) + ReportUploadService.finalize(report_upload.id, result) return Response( build_upload_success_payload( form="f5", report_year=report_year, status="done", result=FormF5ParseResultSerializer(result).data, + upload_id=str(report_upload.id), ), status=status.HTTP_200_OK, ) except Exception as e: + ReportUploadService.mark_failed(report_upload.id) logger.exception("Ошибка обработки файла Ф-5") return build_upload_error_response( error_code="processing_error", diff --git a/src/apps/form_5/tasks.py b/src/apps/form_5/tasks.py index 1937261..8843d5a 100644 --- a/src/apps/form_5/tasks.py +++ b/src/apps/form_5/tasks.py @@ -3,6 +3,7 @@ import logging from io import BytesIO +from apps.core.report_uploads import ReportUploadService from apps.core.tasks import TrackedTask from apps.form_5.services import FormF5Parser from celery import shared_task @@ -17,10 +18,20 @@ def process_form_f5_file( file_name: str, report_year: int, report_quarter: int | None = None, + report_upload_id: str | None = None, ) -> dict: logger.info(f"Начало обработки файла Ф-5: {file_name}") + if report_upload_id is not None: + ReportUploadService.mark_processing(report_upload_id) parser = FormF5Parser(report_year=report_year, report_quarter=report_quarter) - result = parser.parse(BytesIO(file_content)) + try: + result = parser.parse(BytesIO(file_content)) + if report_upload_id is not None: + ReportUploadService.finalize(report_upload_id, result) + except Exception: + if report_upload_id is not None: + ReportUploadService.mark_failed(report_upload_id) + raise logger.info( f"Обработка Ф-5 завершена: {result.loaded_count} загружено, {result.skipped_count} пропущено" ) diff --git a/src/apps/form_6/api.py b/src/apps/form_6/api.py index de8d18f..9da0a79 100644 --- a/src/apps/form_6/api.py +++ b/src/apps/form_6/api.py @@ -2,6 +2,7 @@ import logging +from apps.core.report_uploads import ReportUploadService from apps.core.upload_contracts import ( build_upload_error_response, build_upload_success_payload, @@ -49,6 +50,13 @@ class FormF6UploadView(APIView): file = serializer.validated_data["file"] report_year = serializer.validated_data["report_year"] report_quarter = serializer.validated_data.get("report_quarter") + report_upload = ReportUploadService.create( + uploaded_file=file, + form="f6", + user=request.user, + report_year=report_year, + report_quarter=report_quarter, + ) if file.size > BACKGROUND_THRESHOLD: task = process_form_f6_file.delay( @@ -56,13 +64,16 @@ class FormF6UploadView(APIView): file.name, report_year, report_quarter, + str(report_upload.id), ) + ReportUploadService.mark_queued(report_upload.id, job_id=task.id) return Response( build_upload_success_payload( form="f6", report_year=report_year, status="queued", job_id=task.id, + upload_id=str(report_upload.id), ), status=status.HTTP_202_ACCEPTED, ) @@ -73,16 +84,19 @@ class FormF6UploadView(APIView): report_year=report_year, report_quarter=report_quarter, ) + ReportUploadService.finalize(report_upload.id, result) return Response( build_upload_success_payload( form="f6", report_year=report_year, status="done", result=FormF6ParseResultSerializer(result).data, + upload_id=str(report_upload.id), ), status=status.HTTP_200_OK, ) except Exception as e: + ReportUploadService.mark_failed(report_upload.id) logger.exception("Ошибка обработки файла Ф-6") return build_upload_error_response( error_code="processing_error", diff --git a/src/apps/form_6/tasks.py b/src/apps/form_6/tasks.py index 7c88e57..3fe45f5 100644 --- a/src/apps/form_6/tasks.py +++ b/src/apps/form_6/tasks.py @@ -3,6 +3,7 @@ import logging from io import BytesIO +from apps.core.report_uploads import ReportUploadService from apps.core.tasks import TrackedTask from apps.form_6.services import FormF6Parser from celery import shared_task @@ -17,10 +18,20 @@ def process_form_f6_file( file_name: str, report_year: int, report_quarter: int | None = None, + report_upload_id: str | None = None, ) -> dict: logger.info(f"Начало обработки файла Ф-6: {file_name}") + if report_upload_id is not None: + ReportUploadService.mark_processing(report_upload_id) parser = FormF6Parser(report_year=report_year, report_quarter=report_quarter) - result = parser.parse(BytesIO(file_content)) + try: + result = parser.parse(BytesIO(file_content)) + if report_upload_id is not None: + ReportUploadService.finalize(report_upload_id, result) + except Exception: + if report_upload_id is not None: + ReportUploadService.mark_failed(report_upload_id) + raise logger.info( f"Обработка Ф-6 завершена: {result.loaded_count} загружено, {result.skipped_count} пропущено" ) diff --git a/src/apps/organization/analytics_services.py b/src/apps/organization/analytics_services.py index fdfe6ca..1a88986 100644 --- a/src/apps/organization/analytics_services.py +++ b/src/apps/organization/analytics_services.py @@ -77,6 +77,16 @@ def _delta_percent(current, previous) -> float: ) +def _direction(current, previous) -> str: + current_value = _dec(current) + previous_value = _dec(previous) + if current_value > previous_value: + return "up" + if current_value < previous_value: + return "down" + return "unchanged" + + def _period_rank(report_quarter: int | None) -> int: return 5 if report_quarter is None else report_quarter @@ -216,14 +226,30 @@ class OrganizationAnalyticsService: *, organization: Organization, report_year: int, - report_quarter: int | None, + report_quarter: int | None = None, ) -> dict[str, object]: f2_records = cls._f2_records(organization) f1_records = cls._f1_records(organization) - current_f2 = cls._require_record( - _pick_record(f2_records, report_year, report_quarter), - entity="Financial summary", - ) + current_f2 = _pick_record(f2_records, report_year, report_quarter) + if current_f2 is None: + empty_metric = { + "amount": 0, + "previous_amount": 0, + "delta_percent": 0.0, + } + return { + "organization_id": str(organization.id), + "report_period": { + "year": report_year, + "quarter": report_quarter, + }, + "data_available": False, + "message": "Данные отсутствуют", + "revenue": empty_metric, + "net_profit": empty_metric.copy(), + "taxes_paid": empty_metric.copy(), + "insurance_contributions": empty_metric.copy(), + } current_f1 = _pick_record(f1_records, report_year, report_quarter) previous_f2 = _pick_record(f2_records, report_year - 1, report_quarter) previous_f1 = _pick_record(f1_records, report_year - 1, report_quarter) @@ -252,6 +278,8 @@ class OrganizationAnalyticsService: "year": report_year, "quarter": report_quarter, }, + "data_available": True, + "message": None, "revenue": { "amount": _amount(current_f2.revenue), "previous_amount": _amount(revenue_previous), @@ -279,7 +307,12 @@ class OrganizationAnalyticsService: @staticmethod def _economics_metric_groups() -> dict[str, tuple[str, ...]]: return { - "efficiency": ("revenue", "ebitda", "net_profit"), + "efficiency": ( + "revenue", + "ebitda", + "net_profit", + "revenue_per_employee", + ), "profitability": ("gross_profit", "net_profit", "operating_profit"), "debt": ("net_debt", "loans", "assets"), "investment": ("capex", "rd_expenses", "assets"), @@ -298,6 +331,7 @@ class OrganizationAnalyticsService: "assets": "rub_thousands", "capex": "rub_thousands", "rd_expenses": "rub_thousands", + "revenue_per_employee": "rub_per_employee", } @staticmethod @@ -341,6 +375,17 @@ class OrganizationAnalyticsService: primary, fallback = field_map.get(metric, (ZERO, ZERO)) return _dec(primary or fallback) + @classmethod + def _economics_display_value(cls, metric: str, f2, f4, f1) -> int: + if metric == "revenue_per_employee": + employees = _dec(getattr(f1, "avg_payroll_employees", ZERO)) + if employees == ZERO: + employees = _dec(getattr(f1, "avg_employees", ZERO)) + if employees == ZERO: + return 0 + return _amount(cls._economics_metric_value("revenue", f2, f4) / employees) + return _amount_thousands(cls._economics_metric_value(metric, f2, f4)) + @classmethod def get_economics( cls, @@ -356,46 +401,58 @@ class OrganizationAnalyticsService: f4_by_year = _best_records_by_year( cls._f4_records(organization), from_year, to_year ) + f1_by_year = _best_records_by_year( + cls._f1_records(organization), from_year, to_year + ) - periods = sorted(set(f2_by_year) | set(f4_by_year)) + periods = sorted(set(f2_by_year) | set(f4_by_year) | set(f1_by_year)) if not periods: - latest_year = max( - filter( - None, - ( - _latest_report_year(cls._f2_records(organization)), - _latest_report_year(cls._f4_records(organization)), - ), - ), - default=None, - ) - if latest_year is None: - raise NotFoundError(message="Economics data is not available") - - f2_by_year = _best_records_by_year( - cls._f2_records(organization), latest_year, latest_year - ) - f4_by_year = _best_records_by_year( - cls._f4_records(organization), latest_year, latest_year - ) - periods = sorted(set(f2_by_year) | set(f4_by_year)) + return { + "organization_id": str(organization.id), + "group": group, + "periods": [], + "data_available": False, + "message": "Данные отсутствуют", + "kpis": {}, + "series": [], + "ratios": [], + "ratio_normatives": ECONOMICS_RATIO_NORMATIVES, + } metric_units = cls._economics_metric_units() selected_metrics = cls._economics_metric_groups()[group] last_period = periods[-1] + previous_period = periods[-2] if len(periods) > 1 else None + + def metric_value(metric: str, period: int | None) -> int: + if period is None: + return 0 + return cls._economics_display_value( + metric, + f2_by_year.get(period), + f4_by_year.get(period), + f1_by_year.get(period), + ) return { "organization_id": str(organization.id), "group": group, "periods": periods, + "data_available": True, + "message": None, "kpis": { metric: { - "value": _amount_thousands( - cls._economics_metric_value( - metric, - f2_by_year.get(last_period), - f4_by_year.get(last_period), - ) + "value": metric_value(metric, last_period), + "previous_value": metric_value(metric, previous_period), + "delta_value": metric_value(metric, last_period) + - metric_value(metric, previous_period), + "delta_percent": _delta_percent( + metric_value(metric, last_period), + metric_value(metric, previous_period), + ), + "direction": _direction( + metric_value(metric, last_period), + metric_value(metric, previous_period), ), "unit": metric_units[metric], } @@ -408,13 +465,7 @@ class OrganizationAnalyticsService: "points": [ { "period": report_year, - "value": _amount_thousands( - cls._economics_metric_value( - metric, - f2_by_year.get(report_year), - f4_by_year.get(report_year), - ) - ), + "value": metric_value(metric, report_year), } for report_year in periods ], @@ -462,6 +513,9 @@ class OrganizationAnalyticsService: current_f1 = _pick_record(cls._f1_records(organization), report_year) years = list(range(report_year - history_years + 1, report_year + 1)) history_records = _best_records_by_year(f3_records, years[0], years[-1]) + history_f1_records = _best_records_by_year( + cls._f1_records(organization), years[0], years[-1] + ) average_employees = int(_dec(current_f3.avg_employees)) under_30 = int(average_employees * 0.29) @@ -494,6 +548,43 @@ class OrganizationAnalyticsService: { "year": year, "average_employees": int(_dec(history_records[year].avg_employees)), + "avg_payroll_employees": int( + _dec( + getattr( + history_f1_records.get(year), + "avg_payroll_employees", + ZERO, + ) + ) + ), + "average_age": _weighted_average_age( + [ + { + "age_group": "under_30", + "employees_count": int( + _dec(history_records[year].avg_employees) + * Decimal("0.29") + ), + }, + { + "age_group": "30_50", + "employees_count": int( + _dec(history_records[year].avg_employees) + * Decimal("0.49") + ), + }, + { + "age_group": "over_50", + "employees_count": int( + _dec(history_records[year].avg_employees) + * Decimal("0.22") + ), + }, + ] + ), + "payroll_fund": _amount( + getattr(history_f1_records.get(year), "payroll_fund", ZERO) + ), } for year in years if year in history_records @@ -592,6 +683,61 @@ class OrganizationAnalyticsService: }, ] + bucket_fields = { + "under_5_years": "age_under_5", + "5_10_years": "age_5_10", + "10_15_years": "age_10_15", + "15_20_years": "age_15_20", + "over_20_years": "age_over_20", + } + total_age_units = sum(item["units_count"] for item in age_distribution) + for item in age_distribution: + bucket = item["bucket"] + if f6_records: + bucket_units_by_record = [ + int(_dec(getattr(record, bucket_fields[bucket], ZERO))) + for record in f6_records + ] + imported_equipment = sum( + round( + int(_dec(record.imported_equipment)) + * units + / max(1, int(_dec(record.total_equipment))) + ) + for record, units in zip( + f6_records, bucket_units_by_record, strict=True + ) + ) + weighted_units = sum(bucket_units_by_record) + utilization_rate = sum( + _dec(record.utilization_rate) * units + for record, units in zip( + f6_records, bucket_units_by_record, strict=True + ) + ) / max(1, weighted_units) + weighted_wear = sum( + _dec(record.physical_wear_percent) * units + for record, units in zip( + f6_records, bucket_units_by_record, strict=True + ) + ) / max(1, weighted_units) + else: + imported_equipment = round( + int(_dec(current_f3.imported_equipment)) + * item["units_count"] + / max(1, total_age_units) + ) + utilization_rate = _dec(current_f3.utilization_rate) + weighted_wear = _dec(current_f3.physical_wear_percent) + item.update( + { + "imported_equipment": imported_equipment, + "utilization_rate": round(float(utilization_rate / 100), 2), + "lease_share_itn_percent": None, + "weighted_wear_percent": _ratio(weighted_wear), + } + ) + if f6_records: category_rows = [ { @@ -661,6 +807,7 @@ class OrganizationAnalyticsService: "report_year": report_year, "summary": { "total_equipment": int(_dec(current_f3.total_equipment)), + "machine_tools_and_equipment": int(_dec(current_f3.total_equipment)), "domestic_equipment": int(_dec(current_f3.domestic_equipment)), "imported_equipment": int(_dec(current_f3.imported_equipment)), "physical_wear_percent": _ratio(current_f3.physical_wear_percent), @@ -1185,7 +1332,7 @@ class DashboardAnalyticsService: def get_dashboard( cls, *, corporation_scope: str | None = None ) -> dict[str, object]: - queryset = Organization.objects.all() + queryset = Organization.objects.filter(mostovik_uid__isnull=False) if corporation_scope: queryset = filter_queryset_by_scopes(queryset, [corporation_scope]) @@ -1234,23 +1381,38 @@ class DashboardAnalyticsService: 1 for org in cluster_organizations if not org.bankruptcy_messages_found ) + cluster_years = sorted( + { + year + for organization in cluster_organizations + for year in f3_best_by_organization.get(str(organization.id), {}) + } + ) + from_year = None + to_year = None growth_values = [] - for organization in cluster_organizations: - organization_f3_by_year = f3_best_by_organization.get( - str(organization.id), {} - ) - report_years = sorted(organization_f3_by_year) - if len(report_years) < 2: - continue - previous_record = organization_f3_by_year[report_years[-2]] - current_record = organization_f3_by_year[report_years[-1]] - if current_record is None or previous_record is None: - continue - growth_values.append( - _delta_percent( - current_record.avg_employees, previous_record.avg_employees + year_pairs = list(zip(cluster_years, cluster_years[1:], strict=False)) + for candidate_from, candidate_to in reversed(year_pairs): + candidate_growth_values = [] + for organization in cluster_organizations: + organization_f3_by_year = f3_best_by_organization.get( + str(organization.id), {} ) - ) + previous_record = organization_f3_by_year.get(candidate_from) + current_record = organization_f3_by_year.get(candidate_to) + if current_record is None or previous_record is None: + continue + candidate_growth_values.append( + _delta_percent( + current_record.avg_employees, + previous_record.avg_employees, + ) + ) + if candidate_growth_values: + from_year = candidate_from + to_year = candidate_to + growth_values = candidate_growth_values + break distribution_by_cluster.append( { @@ -1270,6 +1432,8 @@ class DashboardAnalyticsService: headcount_growth_by_cluster.append( { "cluster": cluster_code, + "from_year": from_year, + "to_year": to_year, "growth_percent": round(sum(growth_values) / len(growth_values), 1) if growth_values else 0.0, diff --git a/src/apps/organization/analytics_views.py b/src/apps/organization/analytics_views.py index af38be5..251111e 100644 --- a/src/apps/organization/analytics_views.py +++ b/src/apps/organization/analytics_views.py @@ -39,7 +39,10 @@ class OrganizationAnalyticsBaseView(APIView): permission_classes = [IsAuthenticated] def get_organization(self, organization_id) -> Organization: - return get_object_or_404(Organization, id=organization_id) + return get_object_or_404( + Organization.objects.filter(mostovik_uid__isnull=False), + id=organization_id, + ) class AnalyticsDashboardView(APIView): diff --git a/src/apps/organization/api.py b/src/apps/organization/api.py index daa1c49..1715715 100644 --- a/src/apps/organization/api.py +++ b/src/apps/organization/api.py @@ -69,14 +69,15 @@ class OrganizationViewSet(ClassicReadOnlyViewSet[Organization]): """ ViewSet для просмотра организаций. - Только чтение - организации создаются автоматически при загрузке форм. + Только чтение. В каталог попадают организации, подтвержденные обменным + пакетом Моставика; строки форм сами по себе каталог не расширяют. Эндпоинты: GET /organizations/ - список организаций GET /organizations/{id}/ - детали организации """ - queryset = Organization.objects.all() + queryset = Organization.objects.filter(mostovik_uid__isnull=False) serializer_class = OrganizationCatalogDetailSerializer permission_classes = [IsAuthenticated] filterset_class = OrganizationFilter diff --git a/src/apps/organization/contract_serializers.py b/src/apps/organization/contract_serializers.py index e739d5d..24311e1 100644 --- a/src/apps/organization/contract_serializers.py +++ b/src/apps/organization/contract_serializers.py @@ -33,6 +33,8 @@ class FinancialSummaryReportPeriodSerializer(serializers.Serializer): class FinancialSummaryResponseSerializer(serializers.Serializer): organization_id = serializers.UUIDField() report_period = FinancialSummaryReportPeriodSerializer() + data_available = serializers.BooleanField() + message = serializers.CharField(allow_null=True) revenue = FinancialSummaryMetricSerializer() net_profit = FinancialSummaryMetricSerializer() taxes_paid = FinancialSummaryMetricSerializer() @@ -41,6 +43,10 @@ class FinancialSummaryResponseSerializer(serializers.Serializer): class EconomicsKpiSerializer(serializers.Serializer): value = serializers.IntegerField() + previous_value = serializers.IntegerField() + delta_value = serializers.IntegerField() + delta_percent = serializers.FloatField() + direction = serializers.ChoiceField(choices=("up", "down", "unchanged")) unit = serializers.CharField() @@ -48,6 +54,7 @@ class EconomicsKpisSerializer(serializers.Serializer): revenue = EconomicsKpiSerializer(required=False) ebitda = EconomicsKpiSerializer(required=False) net_profit = EconomicsKpiSerializer(required=False) + revenue_per_employee = EconomicsKpiSerializer(required=False) gross_profit = EconomicsKpiSerializer(required=False) operating_profit = EconomicsKpiSerializer(required=False) net_debt = EconomicsKpiSerializer(required=False) @@ -87,6 +94,8 @@ class EconomicsResponseSerializer(serializers.Serializer): organization_id = serializers.UUIDField() group = serializers.CharField() periods = serializers.ListField(child=serializers.IntegerField()) + data_available = serializers.BooleanField() + message = serializers.CharField(allow_null=True) kpis = EconomicsKpisSerializer() series = EconomicsMetricSeriesSerializer(many=True) ratios = EconomicsRatioSerializer(many=True) @@ -111,6 +120,9 @@ class PersonnelAgeDistributionSerializer(serializers.Serializer): class PersonnelHistorySerializer(serializers.Serializer): year = serializers.IntegerField() average_employees = serializers.IntegerField() + avg_payroll_employees = serializers.IntegerField() + average_age = serializers.FloatField() + payroll_fund = serializers.IntegerField() class PersonnelResponseSerializer(serializers.Serializer): @@ -124,6 +136,7 @@ class PersonnelResponseSerializer(serializers.Serializer): class EquipmentSummarySerializer(serializers.Serializer): total_equipment = serializers.IntegerField() + machine_tools_and_equipment = serializers.IntegerField() domestic_equipment = serializers.IntegerField() imported_equipment = serializers.IntegerField() physical_wear_percent = serializers.FloatField() @@ -139,6 +152,10 @@ class EquipmentSummarySerializer(serializers.Serializer): class EquipmentAgeDistributionSerializer(serializers.Serializer): bucket = serializers.CharField() units_count = serializers.IntegerField() + imported_equipment = serializers.IntegerField() + utilization_rate = serializers.FloatField() + lease_share_itn_percent = serializers.FloatField(allow_null=True) + weighted_wear_percent = serializers.FloatField() class EquipmentCategorySerializer(serializers.Serializer): @@ -320,6 +337,8 @@ class DashboardExecutorsSerializer(serializers.Serializer): class DashboardHeadcountGrowthSerializer(serializers.Serializer): cluster = serializers.CharField() + from_year = serializers.IntegerField(allow_null=True) + to_year = serializers.IntegerField(allow_null=True) growth_percent = serializers.FloatField() diff --git a/src/apps/organization/management/__init__.py b/src/apps/organization/management/__init__.py new file mode 100644 index 0000000..4ee6e5a --- /dev/null +++ b/src/apps/organization/management/__init__.py @@ -0,0 +1 @@ +"""Organization management commands.""" diff --git a/src/apps/organization/management/commands/__init__.py b/src/apps/organization/management/commands/__init__.py new file mode 100644 index 0000000..0362fa3 --- /dev/null +++ b/src/apps/organization/management/commands/__init__.py @@ -0,0 +1 @@ +"""Organization command package.""" diff --git a/src/apps/organization/management/commands/delete_test_companies.py b/src/apps/organization/management/commands/delete_test_companies.py new file mode 100644 index 0000000..083d424 --- /dev/null +++ b/src/apps/organization/management/commands/delete_test_companies.py @@ -0,0 +1,107 @@ +"""Delete the fixed Mostovik demonstration companies and related data.""" + +from __future__ import annotations + +import json + +from apps.core.management.commands.base import BaseAppCommand +from apps.core.models import ReportUpload +from apps.organization.models import Organization +from django.db import transaction +from django.db.models import Q + +TEST_COMPANY_COUNT = 20 + + +def _legal_entity_inn(index: int) -> str: + base = f"770900{index:03d}" + weights = (2, 4, 10, 3, 5, 9, 4, 6, 8) + checksum = sum( + int(digit) * weight for digit, weight in zip(base, weights, strict=True) + ) + return f"{base}{checksum % 11 % 10}" + + +TEST_COMPANY_IDENTITIES = tuple( + (f"Тестовая компания {index}", _legal_entity_inn(index)) + for index in range(1, TEST_COMPANY_COUNT + 1) +) +DEMO_REPORT_FILE_NAMES = ( + "Ф-1_2022-12.xlsx", + "Ф-1_2023-12.xlsx", + "Ф-1_2024-12.xlsx", + "Ф-1_2025-09.xlsx", + "Ф-1_2025-12.xlsx", + "Ф-1_2026-03.xlsx", + "Ф-1_2026-06.xlsx", + "Ф-2_2025-Q3.xlsx", + "Ф-2_2025-Q4.xlsx", + "Ф-2_2026-Q1.xlsx", + "Ф-2_2026-Q2.xlsx", + "Ф-3_2022.xlsx", + "Ф-3_2023.xlsx", + "Ф-3_2024.xlsx", + "Ф-3_2025.xlsx", + "Ф-4_2024-H2.xlsx", + "Ф-4_2025-H1.xlsx", + "Ф-4_2025-H2.xlsx", + "Ф-4_2026-H1.xlsx", + "Ф-5_2022.xlsx", + "Ф-5_2023.xlsx", + "Ф-5_2024.xlsx", + "Ф-5_2025.xlsx", + "Ф-6_2022.xlsx", + "Ф-6_2023.xlsx", + "Ф-6_2024.xlsx", + "Ф-6_2025.xlsx", +) + + +class Command(BaseAppCommand): + """Delete only the deterministic companies imported from Mostovik.""" + + help = "Удаляет 20 тестовых компаний Mostovik и все связанные данные" + use_transaction = True + + def execute_command(self, *args, **options) -> str: + report_uploads = list( + ReportUpload.objects.filter(file_name__in=DEMO_REPORT_FILE_NAMES) + ) + stored_files = [ + (upload.original_file.storage, upload.original_file.name) + for upload in report_uploads + if upload.original_file and upload.original_file.name + ] + report_uploads_deleted = len(report_uploads) + ReportUpload.objects.filter( + pk__in=[item.pk for item in report_uploads] + ).delete() + for storage, file_name in stored_files: + transaction.on_commit( + lambda storage=storage, file_name=file_name: storage.delete(file_name) + ) + + identity_query = Q() + for name, inn in TEST_COMPANY_IDENTITIES: + identity_query |= Q(name=name, inn=inn) + + queryset = Organization.objects.filter(identity_query) + organizations_deleted = queryset.count() + total_deleted, deleted_by_model = queryset.delete() + organization_label = Organization._meta.label + related_objects_deleted = total_deleted - deleted_by_model.get( + organization_label, 0 + ) + + rendered = json.dumps( + { + "dry_run": self.dry_run, + "organizations_deleted": organizations_deleted, + "report_uploads_deleted": report_uploads_deleted, + "related_objects_deleted": related_objects_deleted, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/apps/user/views.py b/src/apps/user/views.py index b613835..6b7d11e 100644 --- a/src/apps/user/views.py +++ b/src/apps/user/views.py @@ -11,6 +11,7 @@ from rest_framework.decorators import api_view, permission_classes from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView +from rest_framework_simplejwt.exceptions import TokenError from rest_framework_simplejwt.tokens import RefreshToken from .serializers import ( @@ -333,7 +334,7 @@ class TokenRefreshView(APIView): return Response( {"access": str(refresh.access_token), "refresh": str(refresh)} ) - except Exception: + except TokenError: return Response( {"error": "Неверный refresh token"}, status=status.HTTP_401_UNAUTHORIZED ) diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index 37acc04..705b263 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -4,6 +4,8 @@ API v1 URL configuration. Все API эндпоинты версионированы под /api/v1/ """ +from apps.core.report_upload_history import ReportUploadHistoryView +from apps.core.report_uploads import ReportUploadDownloadView from apps.core.views import ( BackgroundJobListView, BackgroundJobStatusView, @@ -34,4 +36,14 @@ urlpatterns = [ path("forms/f4/", include("apps.form_4.urls")), path("forms/f5/", include("apps.form_5.urls")), path("forms/f6/", include("apps.form_6.urls")), + path( + "forms/uploads/history/", + ReportUploadHistoryView.as_view(), + name="report-upload-history", + ), + path( + "forms//uploads//download/", + ReportUploadDownloadView.as_view(), + name="report-upload-download", + ), ] diff --git a/src/settings/base.py b/src/settings/base.py index 8242dda..2599a80 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -227,6 +227,7 @@ USE_TZ = True STATIC_URL = "/static/" STATIC_ROOT = PROJECT_ROOT / "staticfiles" STATICFILES_DIRS = [BASE_DIR / "static"] +STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" MEDIA_URL = "/media/" MEDIA_ROOT = PROJECT_ROOT / "media" @@ -267,8 +268,10 @@ REST_FRAMEWORK = { SIMPLE_JWT = { "ACCESS_TOKEN_LIFETIME": timedelta(minutes=60), "REFRESH_TOKEN_LIFETIME": timedelta(days=7), - "ROTATE_REFRESH_TOKENS": True, - "BLACKLIST_AFTER_ROTATION": True, + # The frontend can issue concurrent refresh requests. Reusing one refresh token + # avoids a race where the first response blacklists the token used by the second. + "ROTATE_REFRESH_TOKENS": False, + "BLACKLIST_AFTER_ROTATION": False, "UPDATE_LAST_LOGIN": True, "ALGORITHM": "HS256", "VERIFYING_KEY": None, diff --git a/src/settings/test.py b/src/settings/test.py index 5ea45ec..7d6ef01 100644 --- a/src/settings/test.py +++ b/src/settings/test.py @@ -107,5 +107,6 @@ SIMPLE_JWT = { **globals().get("SIMPLE_JWT", {}), "ACCESS_TOKEN_LIFETIME": timedelta(minutes=5), "REFRESH_TOKEN_LIFETIME": timedelta(days=1), - "ROTATE_REFRESH_TOKENS": True, + "ROTATE_REFRESH_TOKENS": False, + "BLACKLIST_AFTER_ROTATION": False, } diff --git a/tests/apps/exchange/test_api.py b/tests/apps/exchange/test_api.py index 44c48bc..10366cb 100644 --- a/tests/apps/exchange/test_api.py +++ b/tests/apps/exchange/test_api.py @@ -371,6 +371,7 @@ class ExchangePackageApiTest(APITestCase): def setUp(self): self.url = reverse("api_v1:exchange:package-upload") + self.latest_url = reverse("api_v1:exchange:package-latest") password = get_random_string(16) self.user = User.objects.create_user( username="exchange-admin", @@ -378,6 +379,55 @@ class ExchangePackageApiTest(APITestCase): password=password, ) + def test_latest_requires_authentication(self): + response = self.client.get(self.latest_url) + + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_latest_returns_null_when_no_packages_exist(self): + self.client.force_authenticate(self.user) + + response = self.client.get(self.latest_url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.data, + {"success": True, "data": None, "errors": None, "meta": None}, + ) + + def test_latest_returns_newest_package_metadata(self): + older = ExchangePackageImport.objects.create( + package_id="older-package", + package_name="older.zip", + package_hash="a" * 64, + source_system="mostovik", + schema_version=1, + delivery_channel=ExchangeDeliveryChannel.API, + status="failed", + ) + latest = ExchangePackageImport.objects.create( + package_id="latest-package", + package_name="latest.zip", + package_hash="b" * 64, + source_system="mostovik", + schema_version=1, + delivery_channel=ExchangeDeliveryChannel.API, + status="success", + ) + ExchangePackageImport.objects.filter(pk=older.pk).update( + created_at=older.created_at.replace(year=older.created_at.year - 1) + ) + self.client.force_authenticate(self.user) + + response = self.client.get(self.latest_url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["data"]["id"], str(latest.id)) + self.assertEqual(response.data["data"]["package_id"], "latest-package") + self.assertEqual(response.data["data"]["package_name"], "latest.zip") + self.assertEqual(response.data["data"]["status"], "success") + self.assertIn("created_at", response.data["data"]) + def test_upload_imports_package_and_upserts_models(self): Organization.objects.create( inn="7707083893", diff --git a/tests/apps/forms/test_report_uploads_api.py b/tests/apps/forms/test_report_uploads_api.py new file mode 100644 index 0000000..05f8ef2 --- /dev/null +++ b/tests/apps/forms/test_report_uploads_api.py @@ -0,0 +1,173 @@ +"""Tests for persistent original report uploads and binary downloads.""" + +import hashlib +import tempfile + +from apps.core.models import ReportUpload, ReportUploadStatus +from apps.core.report_uploads import ReportUploadService +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import override_settings +from django.urls import reverse +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.form_1.factories import FormF1RecordFactory +from tests.apps.organization.factories import OrganizationFactory +from tests.apps.user.factories import UserFactory + + +class ReportUploadApiTest(APITestCase): + def setUp(self): + self.media_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.media_directory.cleanup) + self.override = override_settings(MEDIA_ROOT=self.media_directory.name) + self.override.enable() + self.addCleanup(self.override.disable) + self.user = UserFactory.create_user() + + def _create_upload(self, *, content=b"original excel bytes") -> ReportUpload: + return ReportUploadService.create( + uploaded_file=SimpleUploadedFile( + "Ф-1 отчет.xlsx", + content, + content_type="application/octet-stream", + ), + form="f1", + user=self.user, + report_year=2026, + report_month=9, + ) + + def test_service_persists_original_bytes_and_finalizes_batch(self): + content = b"same bytes that arrived" + report_upload = self._create_upload(content=content) + + ReportUploadService.finalize(report_upload.id, {"batch_id": 42}) + report_upload.refresh_from_db() + + self.assertEqual(report_upload.load_batch, 42) + self.assertEqual(report_upload.status, ReportUploadStatus.SUCCESS) + self.assertEqual(report_upload.file_hash, hashlib.sha256(content).hexdigest()) + self.assertEqual(report_upload.original_file.read(), content) + self.assertEqual(report_upload.file_name, "Ф-1 отчет.xlsx") + self.assertEqual(report_upload.uploaded_by, self.user) + + def test_download_requires_authentication(self): + report_upload = self._create_upload() + ReportUploadService.finalize(report_upload.id, {"batch_id": 42}) + + response = self.client.get( + reverse("api_v1:report-upload-download", args=["f1", 42]) + ) + + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_download_streams_original_bytes_and_filename(self): + content = b"unchanged original xlsx bytes" + report_upload = self._create_upload(content=content) + ReportUploadService.finalize(report_upload.id, {"batch_id": 42}) + self.client.force_authenticate(self.user) + + response = self.client.get( + reverse("api_v1:report-upload-download", args=["f1", 42]) + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(b"".join(response.streaming_content), content) + self.assertEqual( + response["Content-Type"], + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + self.assertIn("attachment", response["Content-Disposition"]) + self.assertIn("utf-8", response["Content-Disposition"].lower()) + + def test_download_rejects_a_different_non_staff_user(self): + report_upload = self._create_upload() + ReportUploadService.finalize(report_upload.id, {"batch_id": 42}) + self.client.force_authenticate(UserFactory.create_user()) + + response = self.client.get( + reverse("api_v1:report-upload-download", args=["f1", 42]) + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + self.assertEqual(response.data["errors"][0]["code"], "upload_forbidden") + + def test_download_returns_gone_when_storage_object_was_removed(self): + report_upload = self._create_upload() + ReportUploadService.finalize(report_upload.id, {"batch_id": 42}) + report_upload.original_file.storage.delete(report_upload.original_file.name) + self.client.force_authenticate(self.user) + + response = self.client.get( + reverse("api_v1:report-upload-download", args=["f1", 42]) + ) + + self.assertEqual(response.status_code, status.HTTP_410_GONE) + self.assertEqual(response.data["errors"][0]["code"], "source_file_gone") + + def test_history_returns_upload_before_records_are_parsed(self): + report_upload = self._create_upload() + ReportUploadService.mark_queued(report_upload.id, job_id="job-42") + self.client.force_authenticate(self.user) + + response = self.client.get("/api/v1/forms/uploads/history/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + row = response.data["results"][0] + self.assertEqual(row["upload_id"], str(report_upload.id)) + self.assertEqual(row["form"], "f1") + self.assertEqual(row["report_year"], 2026) + self.assertEqual(row["report_month"], 9) + self.assertEqual(row["status"], "queued") + self.assertEqual(row["job_id"], "job-42") + self.assertIsNone(row["organization_id"]) + self.assertFalse(row["download_available"]) + + def test_history_projects_successful_upload_per_organization(self): + organization = OrganizationFactory.create( + name="Тестовая компания 1", + inn="7709000010", + gk_code="rosatom", + gk_name="Госкорпорация Росатом", + ) + report_upload = self._create_upload() + FormF1RecordFactory.create( + organization=organization, + load_batch=42, + report_year=2026, + report_month=9, + ) + ReportUploadService.finalize(report_upload.id, {"batch_id": 42}) + self.client.force_authenticate(self.user) + + response = self.client.get( + "/api/v1/forms/uploads/history/", + { + "corporation_scope": "rosatom", + "organization_inn": organization.inn, + "report_year": 2026, + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 1) + row = response.data["results"][0] + self.assertEqual(row["organization_id"], str(organization.id)) + self.assertEqual(row["organization_name"], organization.name) + self.assertEqual(row["organization_inn"], organization.inn) + self.assertEqual(row["corporation_scope"], "rosatom") + self.assertEqual(row["report_period_display"], "Сентябрь 2026") + self.assertEqual(row["records_count"], 1) + self.assertTrue(row["download_available"]) + self.assertEqual(row["download_url"], "/api/v1/forms/f1/uploads/42/download/") + + def test_history_does_not_expose_another_users_uploads(self): + self._create_upload() + self.client.force_authenticate(UserFactory.create_user()) + + response = self.client.get("/api/v1/forms/uploads/history/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["count"], 0) diff --git a/tests/apps/forms/test_upload_contracts_api.py b/tests/apps/forms/test_upload_contracts_api.py index 3a9f71b..72241d3 100644 --- a/tests/apps/forms/test_upload_contracts_api.py +++ b/tests/apps/forms/test_upload_contracts_api.py @@ -2,10 +2,12 @@ from __future__ import annotations +import tempfile from datetime import datetime from types import SimpleNamespace from unittest.mock import Mock, patch +from apps.core.models import ReportUpload, ReportUploadStatus from django.core.files.uploadedfile import SimpleUploadedFile from django.test import override_settings from rest_framework import status @@ -89,6 +91,11 @@ class FormUploadContractsApiTest(APITestCase): } def setUp(self): + self.media_directory = tempfile.TemporaryDirectory() + self.addCleanup(self.media_directory.cleanup) + self.media_override = override_settings(MEDIA_ROOT=self.media_directory.name) + self.media_override.enable() + self.addCleanup(self.media_override.disable) self.user = UserFactory.create_user() self.client.force_authenticate(self.user) @@ -172,6 +179,21 @@ class FormUploadContractsApiTest(APITestCase): ) parse_mock.assert_not_called() task_mock.delay.assert_called_once() + task_args, task_kwargs = task_mock.delay.call_args + self.assertIn( + response.data["upload_id"], + (*task_args, *task_kwargs.values()), + ) + report_upload = ReportUpload.objects.get( + pk=response.data["upload_id"] + ) + self.assertEqual(report_upload.form, case["form"]) + self.assertEqual(report_upload.status, ReportUploadStatus.QUEUED) + self.assertIsNone(report_upload.load_batch) + self.assertEqual( + report_upload.original_file.read(), + b"0" * self.BACKGROUND_THRESHOLD_PLUS, + ) def test_upload_sync_contract(self): for _, case in self.CASES.items(): @@ -201,6 +223,13 @@ class FormUploadContractsApiTest(APITestCase): ) parse_mock.assert_called_once() task_mock.delay.assert_not_called() + report_upload = ReportUpload.objects.get( + pk=response.data["upload_id"] + ) + self.assertEqual(report_upload.form, case["form"]) + self.assertEqual(report_upload.load_batch, 101) + self.assertEqual(report_upload.status, ReportUploadStatus.SUCCESS) + self.assertEqual(report_upload.original_file.read(), b"0" * 256) def test_upload_processing_error_contract(self): for _, case in self.CASES.items(): @@ -219,3 +248,7 @@ class FormUploadContractsApiTest(APITestCase): self.assertEqual(response.data["error_message"], "parse failed") self.assertEqual(response.data["details"], []) parse_mock.assert_called_once() + report_upload = ReportUpload.objects.filter(form=case["form"]).latest( + "created_at" + ) + self.assertEqual(report_upload.status, ReportUploadStatus.FAILED) diff --git a/tests/apps/organization/factories.py b/tests/apps/organization/factories.py index 72e9d14..3c6f940 100644 --- a/tests/apps/organization/factories.py +++ b/tests/apps/organization/factories.py @@ -33,6 +33,7 @@ class OrganizationFactory(factory.django.DjangoModelFactory): ogrn = factory.LazyAttribute(lambda _: fake.numerify("#############")) kpp = factory.LazyAttribute(lambda _: fake.numerify("#########")) okpo = factory.LazyAttribute(lambda _: fake.numerify("########")) + mostovik_uid = factory.Faker("uuid4") registration_date = factory.LazyAttribute(lambda _: fake.date_this_century()) legal_address = factory.LazyAttribute(lambda _: fake.address().replace("\n", ", ")) activity_type = factory.LazyAttribute(lambda _: fake.job()) diff --git a/tests/apps/organization/test_analytics_api.py b/tests/apps/organization/test_analytics_api.py index ce4a918..8f3ada8 100644 --- a/tests/apps/organization/test_analytics_api.py +++ b/tests/apps/organization/test_analytics_api.py @@ -194,6 +194,32 @@ class OrganizationAnalyticsApiTest(APITestCase): set(response.data["insurance_contributions"]), {"amount", "previous_amount", "delta_percent"}, ) + self.assertTrue(response.data["data_available"]) + self.assertIsNone(response.data["message"]) + + def test_financial_summary_returns_explicit_empty_state(self): + organization = OrganizationFactory.create() + + response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/financial-summary/" + "?report_year=2026&report_quarter=3" + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertFalse(response.data["data_available"]) + self.assertEqual(response.data["message"], "Данные отсутствуют") + self.assertEqual(response.data["revenue"]["amount"], 0) + + def test_financial_summary_allows_annual_request_without_quarter(self): + response = self.client.get( + f"/api/v1/organizations/{self.organization.id}/analytics/financial-summary/" + "?report_year=2026" + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + response.data["report_period"], {"year": 2026, "quarter": None} + ) def test_economics_contract(self): response = self.client.get( @@ -207,14 +233,29 @@ class OrganizationAnalyticsApiTest(APITestCase): self.assertEqual(response.data["periods"], [2025, 2026]) self.assertEqual( response.data["kpis"].keys(), - {"revenue", "ebitda", "net_profit"}, + {"revenue", "ebitda", "net_profit", "revenue_per_employee"}, ) + self.assertGreater(response.data["kpis"]["revenue_per_employee"]["value"], 0) + self.assertEqual( + response.data["kpis"]["revenue_per_employee"]["unit"], + "rub_per_employee", + ) + for kpi in response.data["kpis"].values(): + self.assertIn("previous_value", kpi) + self.assertIn("delta_value", kpi) + self.assertIn("delta_percent", kpi) + self.assertIn("direction", kpi) self.assertIn("series", response.data) for series in response.data["series"]: self.assertIn("metric", series) self.assertIn("unit", series) self.assertIn("points", series) - self.assertEqual(series["unit"], "rub_thousands") + expected_unit = ( + "rub_per_employee" + if series["metric"] == "revenue_per_employee" + else "rub_thousands" + ) + self.assertEqual(series["unit"], expected_unit) self.assertEqual(len(series["points"]), 2) self.assertEqual(set(series["points"][0]), {"period", "value"}) @@ -234,15 +275,17 @@ class OrganizationAnalyticsApiTest(APITestCase): ) ) - def test_economics_uses_latest_available_year_when_requested_range_is_empty(self): + def test_economics_respects_requested_range_when_it_is_empty(self): response = self.client.get( f"/api/v1/organizations/{self.organization.id}/analytics/economics/" "?group=efficiency&from_year=2022&to_year=2024" ) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data["periods"], [2026]) - self.assertEqual(response.data["series"][0]["points"][0]["period"], 2026) + self.assertEqual(response.data["periods"], []) + self.assertEqual(response.data["kpis"], {}) + self.assertFalse(response.data["data_available"]) + self.assertEqual(response.data["message"], "Данные отсутствуют") def test_personnel_contract(self): personnel_response = self.client.get( @@ -259,6 +302,16 @@ class OrganizationAnalyticsApiTest(APITestCase): 1050, ) self.assertEqual(len(personnel_response.data["history"]), 2) + self.assertEqual( + set(personnel_response.data["history"][0]), + { + "year", + "average_employees", + "avg_payroll_employees", + "average_age", + "payroll_fund", + }, + ) self.assertEqual(len(personnel_response.data["age_distribution"]), 3) self.assertIn("average_age", personnel_response.data) self.assertEqual( @@ -307,6 +360,7 @@ class OrganizationAnalyticsApiTest(APITestCase): set(response.data["summary"]), { "total_equipment", + "machine_tools_and_equipment", "domestic_equipment", "imported_equipment", "physical_wear_percent", @@ -329,6 +383,17 @@ class OrganizationAnalyticsApiTest(APITestCase): response.data["age_distribution"][0]["bucket"], "under_5_years" ) self.assertEqual(len(response.data["age_distribution"]), 5) + self.assertEqual( + set(response.data["age_distribution"][0]), + { + "bucket", + "units_count", + "imported_equipment", + "utilization_rate", + "lease_share_itn_percent", + "weighted_wear_percent", + }, + ) self.assertGreaterEqual(len(response.data["categories"]), 1) self.assertEqual(len(response.data["dynamics_series"]), 3) self.assertEqual( @@ -608,6 +673,42 @@ class OrganizationAnalyticsApiTest(APITestCase): self.assertEqual( response.data["headcount_growth_by_cluster"][0]["growth_percent"], 50.0 ) + self.assertEqual( + response.data["headcount_growth_by_cluster"][0]["from_year"], 2023 + ) + self.assertEqual( + response.data["headcount_growth_by_cluster"][0]["to_year"], 2024 + ) + + def test_dashboard_growth_ignores_latest_year_without_comparable_history(self): + comparable = OrganizationFactory.create( + cluster="space", + gk_code="1", + gk_name='Госкорпорация "Роскосмос"', + ) + latest_only = OrganizationFactory.create( + cluster="space", + gk_code="1", + gk_name='Госкорпорация "Роскосмос"', + ) + FormF3RecordFactory.create( + organization=comparable, report_year=2023, avg_employees=100 + ) + FormF3RecordFactory.create( + organization=comparable, report_year=2024, avg_employees=120 + ) + FormF3RecordFactory.create( + organization=latest_only, report_year=2025, avg_employees=200 + ) + + response = self.client.get( + "/api/v1/analytics/dashboard/?corporation_scope=roskosmos" + ) + + growth = response.data["headcount_growth_by_cluster"][0] + self.assertEqual(growth["from_year"], 2023) + self.assertEqual(growth["to_year"], 2024) + self.assertEqual(growth["growth_percent"], 20.0) def test_analytics_query_validation(self): response = self.client.get( diff --git a/tests/apps/organization/test_api.py b/tests/apps/organization/test_api.py index fcfa9bd..85e69be 100644 --- a/tests/apps/organization/test_api.py +++ b/tests/apps/organization/test_api.py @@ -43,6 +43,25 @@ class OrganizationApiTest(APITestCase): self.assertEqual(response.data["results"][0]["registry_category_label"], "ОПК") self.assertEqual(response.data["results"][0]["short_name"], "АО «Альфа»") + def test_catalog_hides_form_only_organization_until_mostovik_confirms_it(self): + organization = OrganizationFactory.create(mostovik_uid=None) + + list_response = self.client.get("/api/v1/organizations/") + detail_response = self.client.get(f"/api/v1/organizations/{organization.id}/") + + self.assertEqual(list_response.status_code, status.HTTP_200_OK) + self.assertEqual(list_response.data["count"], 0) + self.assertEqual(detail_response.status_code, status.HTTP_404_NOT_FOUND) + + organization.mostovik_uid = "11111111-1111-4111-8111-111111111111" + organization.save(update_fields=["mostovik_uid", "updated_at"]) + + list_response = self.client.get("/api/v1/organizations/") + detail_response = self.client.get(f"/api/v1/organizations/{organization.id}/") + + self.assertEqual(list_response.data["count"], 1) + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + def test_detail_includes_active_registries(self): organization = OrganizationFactory.create( short_name="АО «Бета»", diff --git a/tests/apps/organization/test_delete_test_companies_command.py b/tests/apps/organization/test_delete_test_companies_command.py new file mode 100644 index 0000000..252af66 --- /dev/null +++ b/tests/apps/organization/test_delete_test_companies_command.py @@ -0,0 +1,114 @@ +"""Tests for deleting the fixed Mostovik demonstration companies.""" + +from __future__ import annotations + +import json +from io import StringIO + +from apps.core.models import ReportUpload +from apps.core.report_uploads import ReportUploadService +from apps.external_data.models import IndustrialProduct +from apps.form_1.models import FormF1Record +from apps.organization.models import Organization +from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.management import call_command +from django.test import TestCase + + +class DeleteTestCompaniesCommandTest(TestCase): + """The cleanup command must be narrow, cascading, and idempotent.""" + + def test_deletes_only_exact_test_identities_with_related_data(self): + first = Organization.objects.create( + name="Тестовая компания 1", + inn="7709000010", + ) + second = Organization.objects.create( + name="Тестовая компания 2", + inn="7709000028", + ) + same_pattern_but_not_test_identity = Organization.objects.create( + name="Тестовая компания 99", + inn="7709000099", + ) + same_inn_pattern_but_real_name = Organization.objects.create( + name="Производственная компания", + inn="7709000035", + ) + FormF1Record.objects.create( + organization=first, + load_batch=1, + report_year=2025, + report_month=9, + ) + IndustrialProduct.objects.create( + organization=second, + product_name="Тестовый комплекс", + product_class="Оборудование", + ) + demo_upload = ReportUploadService.create( + uploaded_file=SimpleUploadedFile("Ф-1_2025-09.xlsx", b"demo"), + form="f1", + ) + historical_demo_upload = ReportUploadService.create( + uploaded_file=SimpleUploadedFile("Ф-1_2022-12.xlsx", b"demo-history"), + form="f1", + ) + demo_file_name = demo_upload.original_file.name + demo_storage = demo_upload.original_file.storage + unrelated_upload = ReportUploadService.create( + uploaded_file=SimpleUploadedFile("Ф-1.xlsx", b"unrelated"), + form="f1", + ) + + stdout = StringIO() + with self.captureOnCommitCallbacks(execute=True): + result = call_command("delete_test_companies", stdout=stdout) + payload = json.loads(result) + + self.assertEqual(payload["organizations_deleted"], 2) + self.assertEqual(payload["related_objects_deleted"], 2) + self.assertEqual(payload["report_uploads_deleted"], 2) + self.assertFalse(Organization.objects.filter(pk__in=[first.pk, second.pk])) + self.assertTrue( + Organization.objects.filter(pk=same_pattern_but_not_test_identity.pk) + ) + self.assertTrue( + Organization.objects.filter(pk=same_inn_pattern_but_real_name.pk) + ) + self.assertFalse(FormF1Record.objects.exists()) + self.assertFalse(IndustrialProduct.objects.exists()) + self.assertFalse(ReportUpload.objects.filter(pk=demo_upload.pk).exists()) + self.assertFalse( + ReportUpload.objects.filter(pk=historical_demo_upload.pk).exists() + ) + self.assertFalse(demo_storage.exists(demo_file_name)) + self.assertTrue(ReportUpload.objects.filter(pk=unrelated_upload.pk).exists()) + + repeated = json.loads(call_command("delete_test_companies", stdout=StringIO())) + self.assertEqual(repeated["organizations_deleted"], 0) + self.assertEqual(repeated["related_objects_deleted"], 0) + self.assertEqual(repeated["report_uploads_deleted"], 0) + + def test_dry_run_preserves_test_companies(self): + organization = Organization.objects.create( + name="Тестовая компания 1", + inn="7709000010", + ) + demo_upload = ReportUploadService.create( + uploaded_file=SimpleUploadedFile("Ф-1_2025-09.xlsx", b"demo"), + form="f1", + ) + + result = call_command( + "delete_test_companies", + dry_run=True, + stdout=StringIO(), + ) + + self.assertTrue(Organization.objects.filter(pk=organization.pk).exists()) + self.assertTrue(ReportUpload.objects.filter(pk=demo_upload.pk).exists()) + self.assertTrue( + demo_upload.original_file.storage.exists(demo_upload.original_file.name) + ) + self.assertTrue(json.loads(result)["dry_run"]) diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index abf8d7e..153e05e 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -398,6 +398,17 @@ class TokenRefreshViewTest(APITestCase): # New refresh token should be different # Refresh token may be the same or different depending on implementation + def test_same_refresh_token_can_handle_parallel_refresh_requests(self): + data = {"refresh": self.tokens["refresh"]} + + first_response = self.client.post(self.refresh_url, data, format="json") + second_response = self.client.post(self.refresh_url, data, format="json") + + self.assertEqual(first_response.status_code, status.HTTP_200_OK) + self.assertEqual(second_response.status_code, status.HTTP_200_OK) + self.assertEqual(first_response.data["refresh"], self.tokens["refresh"]) + self.assertEqual(second_response.data["refresh"], self.tokens["refresh"]) + def test_refresh_token_invalid(self): """Test token refresh fails with invalid refresh token""" data = {"refresh": fake.pystr(min_chars=20, max_chars=50)} diff --git a/tests/test_runtime_static.py b/tests/test_runtime_static.py new file mode 100644 index 0000000..eccf7aa --- /dev/null +++ b/tests/test_runtime_static.py @@ -0,0 +1,31 @@ +"""Regression checks for production static-file delivery.""" + +from pathlib import Path + +from django.core.management import call_command +from django.test import Client, override_settings +from settings import base + + +def test_whitenoise_uses_compressed_manifest_storage(): + assert ( + base.STATICFILES_STORAGE + == "whitenoise.storage.CompressedManifestStaticFilesStorage" + ) + + +def test_web_startup_collects_static_files_before_gunicorn(): + script_path = Path(__file__).parents[1] / "docker" / "scripts" / "start-web.sh" + script = script_path.read_text(encoding="utf-8") + + assert "python src/manage.py collectstatic --noinput" in script + assert script.index("collectstatic --noinput") < script.index("exec gunicorn") + + +def test_whitenoise_serves_collected_swagger_asset(tmp_path): + with override_settings(STATIC_ROOT=tmp_path): + call_command("collectstatic", interactive=False, verbosity=0) + response = Client().get("/static/drf-yasg/style.css") + + assert response.status_code == 200 + assert response["Content-Type"].startswith("text/css")