diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 105dd0d..877f7de 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -89,6 +89,7 @@ services: volumes: - ./src:/app/src - ./logs:/app/logs + - ./media:/app/media - ./input:/app/input command: ["celery-worker"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index deed6e9..8c1c974 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -53,6 +53,7 @@ services: memswap_limit: 3g volumes: - ./logs:/app/logs + - ./media:/app/media - ./input:/app/input command: ["celery-worker"] diff --git a/docker/Dockerfile b/docker/Dockerfile index 390a53c..cfdbcbf 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -63,7 +63,10 @@ RUN mkdir -p logs media staticfiles input src/static \ && chown -R appuser:appgroup /app ENV PATH="/app/.venv/bin:${PATH}" \ - PYTHONPATH=/app/src + PYTHONPATH=/app/src \ + SOURCE_RECORD_EXPORT_DIRECTORY=/app/media/source-record-exports \ + SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE=100000 \ + SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS=300 USER appuser ENTRYPOINT ["/app/docker/scripts/entrypoint.sh"] diff --git a/docs/source-record-export-matrix-ru.md b/docs/source-record-export-matrix-ru.md new file mode 100644 index 0000000..f8f4a55 --- /dev/null +++ b/docs/source-record-export-matrix-ru.md @@ -0,0 +1,102 @@ +# Матрица файловых выгрузок внешних данных State Corp + +## Пользовательский контракт + +Администраторский frontend отправляет +`POST /api/v2/organization-source-records/export-ticket/` с массивом `sources` +и форматом. Backend проверяет последнее полностью опубликованное поколение и +возвращает короткоживущий одноразовый ticket. Затем frontend передаёт ticket в +теле обычной HTML-формы на +`POST /api/v2/organization-source-records/export-download/`. + +Имя скачиваемого архива формируется по выбранным источникам и времени создания +запроса: `_YYYYMMDD_HHMMSS.zip`, а для нескольких источников их +имена соединяются через `__`. Значение `file_name` из ticket и итоговый +`Content-Disposition` совпадают. + +Браузер получает потоковый ZIP напрямую, без многогигабайтного `Blob` в +JavaScript. Ticket не попадает в URL и после первого запроса становится +недействительным. Совместимый администраторский endpoint +`POST /api/v2/organization-source-records/export/` сразу возвращает тот же ZIP +для API-клиентов. + +Во время скачивания таблицы `external_data` не читаются: endpoint упаковывает +готовые файлы последнего ночного поколения. При отсутствии поколения API +возвращает `503` с кодом `source_export_not_ready`. + +Каждое поколение содержит только текущий календарный год в timezone сервиса. +Для моделей с предметной датой год определяется по ней, для моделей без такой +даты — по `created_at`, для финансовых отчётов — по году строк отчёта. Строки +других лет из финансового отчёта не выгружаются. После смены года старое +поколение не раздаётся: до успешной сборки нового года API отвечает +`503 source_export_not_ready`. + +## Матрица + +| Группа API | Таблицы State Corp | Файл | CSV | XLSX | JSON | +|---|---|---|:---:|:---:|:---:| +| `financial_indicators` | `FinancialReport`, `FinancialReportLine` | `financial-indicators` | — | — | да | +| `government_procurements` | `PublicProcurement` | `public-procurements` | да | да | да | +| `industrial_production` | `IndustrialProduct`, `IndustrialCertificate`, `ManufacturerRegistryEntry` | `manufacturers-and-products` | да | да | да | +| `planned_inspections` | `ProsecutorCheck` | `planned-inspections` | да | да | да | +| `bankruptcy` | `BankruptcyProcedure` | `bankruptcy-procedures` | да | да | да | +| `defense_suppliers` | `DefenseUnreliableSupplier` | `defense-unreliable-suppliers` | да | да | да | +| `arbitration` | `ArbitrationCase` | `arbitration-cases` | да | да | да | +| `security_registries` | `InformationSecurityRegistryEntry` | `information-security-registries` | да | да | да | +| `vacancies` | `LaborVacancy` | `labor-vacancies` | да | да | да | + +Итого формируется 25 логических артефактов. Финансовые показатели всегда +выгружаются в JSON с вложенным массивом `financial_lines`. Промышленная группа +объединяет три таблицы, а поле `record_type` различает тип строки. Все строки +используют тот же контракт, что и Mostovik: реквизиты организации, включая ОКПО, +общие поля записи источника и специфичные поля в `payload.*`. + +Все записи текущего года включаются в публичные файлы, а техническое +наименование внешнего поставщика нейтрализуется. Исходные значения в БД +сохраняются для работы интеграции и дедупликации. + +Физических XLSX-файлов может быть больше: по умолчанию один файл содержит не +более 100 000 строк данных и получает суффикс `-part-001`, `-part-002` и далее. + +## Ночная генерация + +Celery Beat запускает +`apps.external_data.tasks.refresh_source_record_export_artifacts` ежедневно в +`05:30 Europe/Moscow`. + +Генератор: + +1. читает каждую нормализованную таблицу один раз без model-level сортировки; + предметные даты и fallback по `created_at` индексированы; +2. создаёт компактный JSON-массив и переиспользует его как готовый JSON; +3. потоково формирует CSV и write-only XLSX; +4. записывает календарный `export_year` в manifest; +5. атомарно публикует `current.json` только после готовности всей матрицы; +6. при ошибке удаляет staging и продолжает отдавать предыдущее поколение того же года; +7. сохраняет текущее и предыдущее поколения по умолчанию. + +Web и Celery worker должны использовать общий read-write volume `/app/media`. + +## Первый запуск и настройки + +Первое поколение можно сформировать вручную: + +```bash +PYTHONPATH=src uv run python src/manage.py build_source_record_exports +``` + +| Настройка | Значение по умолчанию | Назначение | +|---|---:|---| +| `SOURCE_RECORD_EXPORT_DIRECTORY` | `media/source-record-exports` | Общий каталог поколений | +| `SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP` | `2` | Число успешных поколений | +| `SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS` | `21600` | TTL распределённой блокировки | +| `SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE` | `100000` | Строк данных в одной XLSX-части | +| `SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS` | `300` | Срок действия download-ticket | + +Для атомарной генерации требуется свободное место не меньше +`(GENERATIONS_TO_KEEP + 1) * размер поколения` плюс запас файловой системы. + +На локальном снимке от 2026-08-04 поколение за 2026 год содержит 1 962 записи +и занимает 4 738 872 байта (4,52 MiB) для всей матрицы. Архив со всеми +источниками оценивается в 2,49 MiB для JSON, 1,59 MiB для CSV и 0,44 MiB для +XLSX; к сумме добавляется небольшой служебный overhead ZIP. diff --git a/input/demo-periods/Ф-1_2026-07.xlsx b/input/demo-periods/Ф-1_2026-07.xlsx new file mode 100644 index 0000000..e78007c Binary files /dev/null and b/input/demo-periods/Ф-1_2026-07.xlsx differ diff --git a/input/demo-periods/Ф-3_2022.xlsx b/input/demo-periods/Ф-3_2022.xlsx index 51d935b..6d20907 100644 Binary files a/input/demo-periods/Ф-3_2022.xlsx and b/input/demo-periods/Ф-3_2022.xlsx differ diff --git a/input/demo-periods/Ф-3_2023.xlsx b/input/demo-periods/Ф-3_2023.xlsx index 9e96cfc..5f6e750 100644 Binary files a/input/demo-periods/Ф-3_2023.xlsx and b/input/demo-periods/Ф-3_2023.xlsx differ diff --git a/input/demo-periods/Ф-3_2024.xlsx b/input/demo-periods/Ф-3_2024.xlsx index be590ea..752b9cc 100644 Binary files a/input/demo-periods/Ф-3_2024.xlsx and b/input/demo-periods/Ф-3_2024.xlsx differ diff --git a/input/demo-periods/Ф-3_2025.xlsx b/input/demo-periods/Ф-3_2025.xlsx index 46bc5f8..297365f 100644 Binary files a/input/demo-periods/Ф-3_2025.xlsx and b/input/demo-periods/Ф-3_2025.xlsx differ diff --git a/input/demo-periods/Ф-3_2026.xlsx b/input/demo-periods/Ф-3_2026.xlsx new file mode 100644 index 0000000..856d961 Binary files /dev/null and b/input/demo-periods/Ф-3_2026.xlsx differ diff --git a/input/demo-periods/Ф-4_2024-H2.xlsx b/input/demo-periods/Ф-4_2024-H2.xlsx index e3faab5..4de29b0 100644 Binary files a/input/demo-periods/Ф-4_2024-H2.xlsx and b/input/demo-periods/Ф-4_2024-H2.xlsx differ diff --git a/input/demo-periods/Ф-4_2025-H1.xlsx b/input/demo-periods/Ф-4_2025-H1.xlsx index e5ec366..a791880 100644 Binary files a/input/demo-periods/Ф-4_2025-H1.xlsx and b/input/demo-periods/Ф-4_2025-H1.xlsx differ diff --git a/input/demo-periods/Ф-4_2025-H2.xlsx b/input/demo-periods/Ф-4_2025-H2.xlsx index 32ac433..ed96971 100644 Binary files a/input/demo-periods/Ф-4_2025-H2.xlsx and b/input/demo-periods/Ф-4_2025-H2.xlsx differ diff --git a/input/demo-periods/Ф-4_2026-H1.xlsx b/input/demo-periods/Ф-4_2026-H1.xlsx index 0004938..8d9a3c9 100644 Binary files a/input/demo-periods/Ф-4_2026-H1.xlsx and b/input/demo-periods/Ф-4_2026-H1.xlsx differ diff --git a/input/demo-periods/Ф-6_2022.xlsx b/input/demo-periods/Ф-6_2022.xlsx index dfb6910..e987131 100644 Binary files a/input/demo-periods/Ф-6_2022.xlsx and b/input/demo-periods/Ф-6_2022.xlsx differ diff --git a/input/demo-periods/Ф-6_2023.xlsx b/input/demo-periods/Ф-6_2023.xlsx index b204cfd..ee2f6f7 100644 Binary files a/input/demo-periods/Ф-6_2023.xlsx and b/input/demo-periods/Ф-6_2023.xlsx differ diff --git a/input/demo-periods/Ф-6_2024.xlsx b/input/demo-periods/Ф-6_2024.xlsx index 96b2c3e..f5d0529 100644 Binary files a/input/demo-periods/Ф-6_2024.xlsx and b/input/demo-periods/Ф-6_2024.xlsx differ diff --git a/input/demo-periods/Ф-6_2025.xlsx b/input/demo-periods/Ф-6_2025.xlsx index 60f7822..bc655bb 100644 Binary files a/input/demo-periods/Ф-6_2025.xlsx and b/input/demo-periods/Ф-6_2025.xlsx differ diff --git a/input/demo-periods/Ф-6_2026.xlsx b/input/demo-periods/Ф-6_2026.xlsx new file mode 100644 index 0000000..3b06e51 Binary files /dev/null and b/input/demo-periods/Ф-6_2026.xlsx differ diff --git a/src/apps/core/excel.py b/src/apps/core/excel.py index c725353..452d8d6 100644 --- a/src/apps/core/excel.py +++ b/src/apps/core/excel.py @@ -290,6 +290,54 @@ class BaseExcelParser(ABC, Generic[T]): """Возвращает следующий номер batch_id.""" raise NotImplementedError + def validate_workbook(self) -> None: + """Validate the loaded workbook schema before any records are created.""" + + @staticmethod + def _normalize_header(value: Any) -> str: + """Normalize whitespace and case for stable Excel header comparisons.""" + if value is None: + return "" + return " ".join(str(value).replace("\xa0", " ").split()).casefold() + + def _validate_mapped_headers( + self, + *, + header_rows: tuple[int, ...] = (1,), + ) -> None: + """Ensure each mapped column still has the expected semantic header.""" + errors: list[FieldError] = [] + for mapping in self._column_mappings or []: + actual = " ".join( + filter( + None, + ( + self._normalize_header( + self._get_cell_value(row, mapping.excel_column) + ) + for row in header_rows + ), + ) + ) + expected = self._normalize_header(mapping.excel_header) + if expected and expected not in actual: + errors.append( + FieldError( + field=mapping.model_field, + message=( + f"Ожидался заголовок «{mapping.excel_header}» " + f"в колонке {mapping.excel_column + 1}" + ), + value=actual or None, + ) + ) + + if errors: + raise ExcelValidationError( + "Схема Excel-файла не соответствует ожидаемому шаблону", + errors=errors, + ) + def parse(self, file: UploadedFile | BytesIO) -> ParseResult: """ Парсит Excel файл и сохраняет данные в БД. @@ -306,6 +354,7 @@ class BaseExcelParser(ABC, Generic[T]): try: self._load_workbook(file) self._column_mappings = self.get_column_mappings() + self.validate_workbook() for row_num in range(self.DATA_START_ROW, self._sheet.max_row + 1): row_data = self._parse_row(row_num) @@ -346,6 +395,8 @@ class BaseExcelParser(ABC, Generic[T]): ) result.skipped_count += 1 + except ExcelValidationError: + raise except Exception as e: logger.exception("Ошибка парсинга Excel файла") raise ExcelParseError(f"Ошибка парсинга файла: {e}") from e diff --git a/src/apps/core/management/commands/generate_test_reports.py b/src/apps/core/management/commands/generate_test_reports.py index d5886b8..83f316f 100644 --- a/src/apps/core/management/commands/generate_test_reports.py +++ b/src/apps/core/management/commands/generate_test_reports.py @@ -605,6 +605,8 @@ class Command(BaseAppCommand): "court_name": "Арбитражный суд города Москвы", "party_role": "defendant" if index % 2 == 0 else "plaintiff", "status": "hearing_scheduled" if index % 4 else "decision_rendered", + "claim_amount": Decimal("500000.00") + + Decimal(index) * Decimal("25000.00"), "decision_date": date( date.today().year, ((index + 2) % 12) + 1, 27 ), @@ -1021,6 +1023,32 @@ class Command(BaseAppCommand): age_over_20 = max( 1, total_equipment - age_under_5 - age_5_10 - age_10_15 - age_15_20 ) + staff_buckets = [ + max(1, int(profile.staff_base * Decimal("0.04"))), + max(1, int(profile.staff_base * Decimal("0.18"))), + max(1, int(profile.staff_base * Decimal("0.28"))), + max(1, int(profile.staff_base * Decimal("0.26"))), + max(1, int(profile.staff_base * Decimal("0.17"))), + ] + staff_buckets.append(max(0, profile.staff_base - sum(staff_buckets))) + average_employee_age = self._rate( + sum( + Decimal(count) * midpoint + for count, midpoint in zip( + staff_buckets, + ( + Decimal("19"), + Decimal("25"), + Decimal("35"), + Decimal("45"), + Decimal("55"), + Decimal("65"), + ), + strict=True, + ) + ) + / Decimal(profile.staff_base) + ) return { "avg_employees": Decimal(profile.staff_base), "production_workers": Decimal(int(profile.staff_base * Decimal("0.64"))), @@ -1031,6 +1059,7 @@ class Command(BaseAppCommand): "total_equipment": total_equipment, "domestic_equipment": domestic_equipment, "imported_equipment": imported_equipment, + "machine_tools_and_equipment": total_equipment, "equipment_age_under_5": age_under_5, "equipment_age_5_10": age_5_10, "equipment_age_10_15": age_10_15, @@ -1043,6 +1072,13 @@ class Command(BaseAppCommand): ), "equipment_needed": max(0, total_equipment // 9), "workers_needed": max(0, profile.staff_base // 22), + "average_employee_age": average_employee_age, + "employees_under_20": staff_buckets[0], + "employees_20_29": staff_buckets[1], + "employees_30_39": staff_buckets[2], + "employees_40_49": staff_buckets[3], + "employees_50_59": staff_buckets[4], + "employees_over_60": staff_buckets[5], } def _build_f4_payload( @@ -1165,13 +1201,32 @@ class Command(BaseAppCommand): age_over_20 = max( 1, total_equipment - age_under_5 - age_5_10 - age_10_15 - age_15_20 ) + age_counts = [ + age_under_5, + age_5_10, + age_10_15, + age_15_20, + age_over_20, + ] + imported_by_age: list[int] = [] + cumulative_equipment = 0 + allocated_imported = 0 + for age_count in age_counts: + cumulative_equipment += age_count + cumulative_imported = ( + imported_equipment * cumulative_equipment // total_equipment + ) + imported_by_age.append(cumulative_imported - allocated_imported) + allocated_imported = cumulative_imported cnc_total = min(total_equipment, max(4, int(total_equipment * Decimal("0.31")))) - return { + payload = { "row_code": f"{100 + (profile.index % 60):03d}", "category": profile.f6_category, "total_equipment": total_equipment, "domestic_equipment": domestic_equipment, "imported_equipment": imported_equipment, + "commissioned_equipment": max(1, total_equipment // 12), + "decommissioned_equipment": max(0, total_equipment // 20), "age_under_5": age_under_5, "age_5_10": age_5_10, "age_10_15": age_10_15, @@ -1190,7 +1245,28 @@ class Command(BaseAppCommand): "physical_wear_percent": wear_percent, "workplaces_without_equipment": max(0, profile.index % 5), "equipment_to_replace": max(0, age_over_20 // 2), + "notes": "Согласованный набор синтетических демо-данных", } + for prefix, imported_count in zip( + ( + "age_under_5", + "age_5_10", + "age_10_15", + "age_15_20", + "age_over_20", + ), + imported_by_age, + strict=True, + ): + payload[f"{prefix}_imported"] = imported_count + payload[f"{prefix}_utilization_rate"] = utilization_rate + payload[f"{prefix}_lease_share_itn_percent"] = self._percent( + Decimal("12") + Decimal(profile.index % 7) + ) + payload[f"{prefix}_weighted_load_factor"] = self._rate( + utilization_rate / Decimal("100") + ) + return payload @staticmethod def _ensure_registers() -> list[Register]: diff --git a/src/apps/core/management/commands/restore_report_upload_files.py b/src/apps/core/management/commands/restore_report_upload_files.py new file mode 100644 index 0000000..70451db --- /dev/null +++ b/src/apps/core/management/commands/restore_report_upload_files.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import hashlib +import json +from collections import defaultdict +from pathlib import Path + +from apps.core.management.commands.base import BaseAppCommand +from apps.core.models import ReportUpload +from django.core.files import File +from django.core.management.base import CommandError + + +def _path_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _storage_sha256(storage, name: str) -> str: + digest = hashlib.sha256() + with storage.open(name, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _build_source_index(source_directories: list[Path]) -> dict[str, list[Path]]: + sources_by_hash: dict[str, list[Path]] = defaultdict(list) + for source_directory in source_directories: + for candidate in sorted(source_directory.rglob("*")): + if candidate.is_file(): + sources_by_hash[_path_sha256(candidate)].append(candidate) + return dict(sources_by_hash) + + +def _restore_upload( + upload: ReportUpload, + *, + sources_by_hash: dict[str, list[Path]], + dry_run: bool, +) -> str: + target_name = str(upload.original_file.name or "").strip() + expected_hash = upload.file_hash.strip().lower() + if not target_name or not expected_hash: + return "unmatched" + + storage = upload.original_file.storage + if storage.exists(target_name): + if _storage_sha256(storage, target_name) == expected_hash: + return "already_present" + return "hash_mismatch" + + candidates = sources_by_hash.get(expected_hash, []) + if not candidates: + return "unmatched" + if dry_run: + return "restored" + + source_path = candidates[0] + with source_path.open("rb") as source_handle: + saved_name = storage.save(target_name, File(source_handle)) + if saved_name != target_name: + storage.delete(saved_name) + raise CommandError( + f"Storage changed target name for report upload {upload.id}." + ) + if _storage_sha256(storage, target_name) != expected_hash: + storage.delete(target_name) + raise CommandError(f"Hash verification failed for report upload {upload.id}.") + return "restored" + + +class Command(BaseAppCommand): + help = ( + "Restore missing original report uploads by matching trusted files by SHA-256." + ) + + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument( + "--source-dir", + action="append", + required=True, + help="Trusted directory to scan recursively; may be supplied multiple times.", + ) + + def execute_command(self, *args, **options) -> str: + source_directories = [Path(value).resolve() for value in options["source_dir"]] + missing_directories = [path for path in source_directories if not path.is_dir()] + if missing_directories: + raise CommandError( + "Source directories do not exist: " + + ", ".join(str(path) for path in missing_directories) + ) + + sources_by_hash = _build_source_index(source_directories) + stats = { + "restored": 0, + "already_present": 0, + "unmatched": 0, + "hash_mismatch": 0, + } + for upload in ReportUpload.objects.order_by("created_at").iterator(): + outcome = _restore_upload( + upload, + sources_by_hash=sources_by_hash, + dry_run=self.dry_run, + ) + stats[outcome] += 1 + + result = json.dumps(stats, ensure_ascii=False, sort_keys=True) + self.log_info(result) + return result diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py index 50bcfe9..444cdfb 100644 --- a/src/apps/exchange/services.py +++ b/src/apps/exchange/services.py @@ -1168,6 +1168,7 @@ class ExchangePackageImportService: skipped_count = 0 created_lines_count = 0 updated_lines_count = 0 + deleted_lines_count = 0 for row in rows: organization = cls._resolve_organization( @@ -1218,6 +1219,7 @@ class ExchangePackageImportService: ) created_lines_count += line_result["created"] updated_lines_count += line_result["updated"] + deleted_lines_count += line_result["deleted"] return { "created": created_count, @@ -1225,6 +1227,7 @@ class ExchangePackageImportService: "skipped": skipped_count, "created_lines": created_lines_count, "updated_lines": updated_lines_count, + "deleted_lines": deleted_lines_count, } @classmethod @@ -1234,7 +1237,7 @@ class ExchangePackageImportService: lines: Any, ) -> dict[str, int]: if lines is None: - return {"created": 0, "updated": 0} + return {"created": 0, "updated": 0, "deleted": 0} if not isinstance(lines, list): raise ExchangeImportError( "Поле lines финансового отчета должно быть списком" @@ -1242,6 +1245,7 @@ class ExchangePackageImportService: created_count = 0 updated_count = 0 + expected_keys: set[tuple[str, str, int]] = set() for line in lines: if not isinstance(line, dict): raise ExchangeImportError( @@ -1252,6 +1256,7 @@ class ExchangePackageImportService: if not form_code or not line_code: continue year = cls._parse_int_value(line.get("year"), field_name="year") + expected_keys.add((form_code, line_code, year)) defaults = { "line_name": cls._clean_string(line.get("line_name")), @@ -1281,7 +1286,22 @@ class ExchangePackageImportService: elif state == "updated": updated_count += 1 - return {"created": created_count, "updated": updated_count} + stale_line_ids = [ + line.pk + for line in report.lines.all() + if (line.form_code, line.line_code, line.year) not in expected_keys + ] + deleted_count = 0 + if stale_line_ids: + deleted_count = FinancialReportLine.objects.filter( + pk__in=stale_line_ids + ).delete()[0] + + return { + "created": created_count, + "updated": updated_count, + "deleted": deleted_count, + } @classmethod def _upsert_arbitration_cases( @@ -1308,6 +1328,11 @@ class ExchangePackageImportService: "court_name": cls._clean_string(row.get("court_name")), "party_role": cls._clean_string(row.get("party_role")), "status": cls._clean_string(row.get("status")), + "claim_amount": cls._parse_decimal_value( + row.get("claim_amount"), + field_name="claim_amount", + allow_null=True, + ), "decision_date": cls._parse_date_value( row.get("decision_date"), field_name="decision_date", diff --git a/src/apps/external_data/export_serializers.py b/src/apps/external_data/export_serializers.py new file mode 100644 index 0000000..f2b22d1 --- /dev/null +++ b/src/apps/external_data/export_serializers.py @@ -0,0 +1,37 @@ +"""Request serializers for prepared external-data exports.""" + +from apps.external_data.source_record_export import ( + EXPORT_FORMATS, + SOURCE_GROUP_EXPORT_SPECS, +) +from rest_framework import serializers + + +class SourceRecordExportRequestSerializer(serializers.Serializer): + """Validate selected source groups and the requested file format.""" + + sources = serializers.ListField( + child=serializers.ChoiceField( + choices=[ + (source_group, source_group) + for source_group in SOURCE_GROUP_EXPORT_SPECS + ] + ), + allow_empty=False, + ) + format = serializers.ChoiceField( + choices=[ + (export_format, export_format.upper()) for export_format in EXPORT_FORMATS + ] + ) + + def validate_sources(self, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise serializers.ValidationError("Источники не должны повторяться.") + return value + + +class SourceRecordExportDownloadSerializer(serializers.Serializer): + """Validate a one-time ticket submitted by a native browser form.""" + + ticket = serializers.CharField(max_length=64, trim_whitespace=False) diff --git a/src/apps/external_data/export_urls.py b/src/apps/external_data/export_urls.py new file mode 100644 index 0000000..fe2a8b9 --- /dev/null +++ b/src/apps/external_data/export_urls.py @@ -0,0 +1,22 @@ +"""URL routes for prepared external-data exports.""" + +from apps.external_data.export_views import ( + SourceRecordExportDownloadView, + SourceRecordExportTicketView, + SourceRecordExportView, +) +from django.urls import path + +app_name = "source_record_exports" + +urlpatterns = [ + path("export/", SourceRecordExportView.as_view(), name="export"), + path( + "export-ticket/", SourceRecordExportTicketView.as_view(), name="export-ticket" + ), + path( + "export-download/", + SourceRecordExportDownloadView.as_view(), + name="export-download", + ), +] diff --git a/src/apps/external_data/export_views.py b/src/apps/external_data/export_views.py new file mode 100644 index 0000000..fa1cd3c --- /dev/null +++ b/src/apps/external_data/export_views.py @@ -0,0 +1,174 @@ +"""HTTP endpoints for prepared external-data exports.""" + +from apps.external_data.export_serializers import ( + SourceRecordExportDownloadSerializer, + SourceRecordExportRequestSerializer, +) +from apps.external_data.source_record_export import ( + SourceRecordExportArchive, + SourceRecordExportArtifactsUnavailable, + SourceRecordExportTicketInvalid, + build_source_records_export_archive, + consume_source_record_export_download_ticket, + create_source_record_export_download_ticket, +) +from django.http import StreamingHttpResponse +from drf_yasg import openapi +from drf_yasg.utils import swagger_auto_schema +from rest_framework import status +from rest_framework.permissions import AllowAny, IsAdminUser +from rest_framework.request import Request +from rest_framework.response import Response +from rest_framework.views import APIView + + +class SourceRecordExportResponseMixin: + """Build the shared streaming ZIP response for export endpoints.""" + + @staticmethod + def source_record_export_response( + package: SourceRecordExportArchive, + ) -> StreamingHttpResponse: + response = StreamingHttpResponse( + package.archive_chunks, + content_type="application/zip", + ) + response[ + "Content-Disposition" + ] = f'attachment; filename="{package.archive_name}"' + response["X-Source-Export-Files"] = str(package.files_count) + response["X-Source-Export-Generated-At"] = package.generated_at + return response + + @staticmethod + def source_record_export_not_ready_response() -> Response: + return Response( + { + "detail": "Готовая ночная выгрузка ещё не сформирована.", + "code": "source_export_not_ready", + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "3600"}, + ) + + +class SourceRecordExportView(SourceRecordExportResponseMixin, APIView): + """Stream a request-specific ZIP from the current prepared generation.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + operation_summary="Выгрузить записи внешних источников", + operation_description=( + "Упаковывает в ZIP готовые файлы выбранных групп без повторного " + "чтения таблиц external_data. Финансовые показатели всегда JSON." + ), + request_body=SourceRecordExportRequestSerializer, + responses={ + 200: openapi.Response( + description="Потоковый ZIP-архив.", + schema=openapi.Schema(type=openapi.TYPE_FILE), + ), + 400: "Некорректные параметры.", + 403: "Доступ разрешён только администраторам.", + 503: "Ночная выгрузка ещё не сформирована.", + }, + tags=["Внешние данные"], + ) + def post(self, request: Request) -> StreamingHttpResponse | Response: + serializer = SourceRecordExportRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + try: + package = build_source_records_export_archive( + source_groups=serializer.validated_data["sources"], + export_format=serializer.validated_data["format"], + ) + except SourceRecordExportArtifactsUnavailable: + return self.source_record_export_not_ready_response() + return self.source_record_export_response(package) + + +class SourceRecordExportTicketView(SourceRecordExportResponseMixin, APIView): + """Issue a short-lived ticket for a native browser download.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + operation_summary="Подготовить нативное скачивание внешних данных", + request_body=SourceRecordExportRequestSerializer, + responses={ + 201: "Одноразовый ticket и имя ZIP-файла.", + 400: "Некорректные параметры.", + 403: "Доступ разрешён только администраторам.", + 503: "Выгрузка или ticket временно недоступны.", + }, + tags=["Внешние данные"], + ) + def post(self, request: Request) -> Response: + serializer = SourceRecordExportRequestSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + try: + download_ticket = create_source_record_export_download_ticket( + source_groups=serializer.validated_data["sources"], + export_format=serializer.validated_data["format"], + ) + except SourceRecordExportArtifactsUnavailable: + return self.source_record_export_not_ready_response() + except RuntimeError: + return Response( + { + "detail": "Не удалось подготовить скачивание. Повторите запрос.", + "code": "source_export_ticket_unavailable", + }, + status=status.HTTP_503_SERVICE_UNAVAILABLE, + headers={"Retry-After": "5"}, + ) + + return Response( + { + "ticket": download_ticket.ticket, + "file_name": download_ticket.archive_name, + "expires_in": download_ticket.expires_in, + }, + status=status.HTTP_201_CREATED, + ) + + +class SourceRecordExportDownloadView(SourceRecordExportResponseMixin, APIView): + """Consume a one-time ticket and stream the prepared ZIP archive.""" + + authentication_classes: list = [] + permission_classes = [AllowAny] + + @swagger_auto_schema( + operation_summary="Скачать готовые внешние данные по ticket", + request_body=SourceRecordExportDownloadSerializer, + responses={ + 200: openapi.Response( + description="Потоковый ZIP-архив.", + schema=openapi.Schema(type=openapi.TYPE_FILE), + ), + 400: "Ticket отсутствует или имеет неверный формат.", + 410: "Ticket истёк или уже использован.", + 503: "Опубликованная выгрузка больше недоступна.", + }, + tags=["Внешние данные"], + ) + def post(self, request: Request) -> StreamingHttpResponse | Response: + serializer = SourceRecordExportDownloadSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + try: + package = consume_source_record_export_download_ticket( + serializer.validated_data["ticket"] + ) + except SourceRecordExportTicketInvalid: + return Response( + { + "detail": "Ticket скачивания истёк или уже использован.", + "code": "source_export_ticket_invalid", + }, + status=status.HTTP_410_GONE, + ) + except SourceRecordExportArtifactsUnavailable: + return self.source_record_export_not_ready_response() + return self.source_record_export_response(package) diff --git a/src/apps/external_data/management/__init__.py b/src/apps/external_data/management/__init__.py new file mode 100644 index 0000000..526fd87 --- /dev/null +++ b/src/apps/external_data/management/__init__.py @@ -0,0 +1 @@ +"""Management package for external-data operations.""" diff --git a/src/apps/external_data/management/commands/__init__.py b/src/apps/external_data/management/commands/__init__.py new file mode 100644 index 0000000..8e6ab52 --- /dev/null +++ b/src/apps/external_data/management/commands/__init__.py @@ -0,0 +1 @@ +"""Management commands for external-data operations.""" diff --git a/src/apps/external_data/management/commands/build_source_record_exports.py b/src/apps/external_data/management/commands/build_source_record_exports.py new file mode 100644 index 0000000..1d8c4ef --- /dev/null +++ b/src/apps/external_data/management/commands/build_source_record_exports.py @@ -0,0 +1,33 @@ +"""Build the complete prepared external-data export matrix.""" + +import json + +from apps.core.management.commands.base import BaseAppCommand +from apps.external_data.source_record_export import ( + build_source_record_export_artifacts, +) + + +class Command(BaseAppCommand): + """Build source-record files synchronously for bootstrap and recovery.""" + + help = "Формирует готовые CSV/XLSX/JSON выгрузки внешних источников" + use_transaction = False + + def execute_command(self, *args, **options) -> str: + generation = build_source_record_export_artifacts() + rendered = json.dumps( + { + "generation_id": generation.generation_id, + "generated_at": generation.generated_at, + "export_year": generation.export_year, + "artifacts_count": generation.artifacts_count, + "files_count": generation.files_count, + "records_count": generation.records_count, + "total_size": generation.total_size, + }, + ensure_ascii=False, + sort_keys=True, + ) + self.log_success(rendered) + return rendered diff --git a/src/apps/external_data/migrations/0007_seed_nightly_source_record_exports.py b/src/apps/external_data/migrations/0007_seed_nightly_source_record_exports.py new file mode 100644 index 0000000..2ea47e5 --- /dev/null +++ b/src/apps/external_data/migrations/0007_seed_nightly_source_record_exports.py @@ -0,0 +1,61 @@ +import json + +from django.db import migrations + +NIGHTLY_SOURCE_EXPORT_TASK_NAME = "external-data:source-record-exports:nightly-msk" +NIGHTLY_SOURCE_EXPORT_TASK_PATH = ( + "apps.external_data.tasks.refresh_source_record_export_artifacts" +) +NIGHTLY_SOURCE_EXPORT_MSK_CRON = { + "minute": "30", + "hour": "5", + "day_of_week": "*", + "day_of_month": "*", + "month_of_year": "*", + "timezone": "Europe/Moscow", +} + + +def seed_nightly_source_record_export_schedule(apps, schema_editor): + CrontabSchedule = apps.get_model("django_celery_beat", "CrontabSchedule") + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + + crontab, _ = CrontabSchedule.objects.get_or_create(**NIGHTLY_SOURCE_EXPORT_MSK_CRON) + field_names = {field.name for field in PeriodicTask._meta.fields} + schedule_fields = {"crontab": crontab} + for field_name in ("interval", "solar", "clocked"): + if field_name in field_names: + schedule_fields[field_name] = None + + PeriodicTask.objects.update_or_create( + name=NIGHTLY_SOURCE_EXPORT_TASK_NAME, + defaults={ + "task": NIGHTLY_SOURCE_EXPORT_TASK_PATH, + "args": json.dumps([]), + "kwargs": json.dumps({}), + "enabled": True, + "description": ( + "Nightly preparation of State Corp external-data export artifacts." + ), + **schedule_fields, + }, + ) + + +def remove_nightly_source_record_export_schedule(apps, schema_editor): + PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask") + PeriodicTask.objects.filter(name=NIGHTLY_SOURCE_EXPORT_TASK_NAME).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("django_celery_beat", "0018_improve_crontab_helptext"), + ("external_data", "0006_bankruptcy_procedure_status_length"), + ] + + operations = [ + migrations.RunPython( + seed_nightly_source_record_export_schedule, + reverse_code=remove_nightly_source_record_export_schedule, + ), + ] diff --git a/src/apps/external_data/migrations/0008_export_date_indexes.py b/src/apps/external_data/migrations/0008_export_date_indexes.py new file mode 100644 index 0000000..0793d30 --- /dev/null +++ b/src/apps/external_data/migrations/0008_export_date_indexes.py @@ -0,0 +1,34 @@ +"""Index the remaining date fields used by current-year exports.""" + +from django.db import migrations, models + + +class Migration(migrations.Migration): + """Add indexes for nullable certificate and security-registry dates.""" + + dependencies = [ + ("external_data", "0007_seed_nightly_source_record_exports"), + ] + + operations = [ + migrations.AlterField( + model_name="industrialcertificate", + name="issue_date", + field=models.DateField( + blank=True, + db_index=True, + null=True, + verbose_name="дата выдачи", + ), + ), + migrations.AlterField( + model_name="informationsecurityregistryentry", + name="issued_at", + field=models.DateField( + blank=True, + db_index=True, + null=True, + verbose_name="дата выдачи", + ), + ), + ] diff --git a/src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py b/src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py new file mode 100644 index 0000000..ffd95ec --- /dev/null +++ b/src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("external_data", "0008_export_date_indexes"), + ] + + operations = [ + migrations.AddField( + model_name="arbitrationcase", + name="claim_amount", + field=models.DecimalField( + blank=True, + decimal_places=2, + max_digits=20, + null=True, + verbose_name="размер иска", + ), + ), + ] diff --git a/src/apps/external_data/migrations/0010_alter_prosecutor_check_control_authority.py b/src/apps/external_data/migrations/0010_alter_prosecutor_check_control_authority.py new file mode 100644 index 0000000..736e319 --- /dev/null +++ b/src/apps/external_data/migrations/0010_alter_prosecutor_check_control_authority.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("external_data", "0009_arbitration_case_claim_amount"), + ] + + operations = [ + migrations.AlterField( + model_name="prosecutorcheck", + name="control_authority", + field=models.TextField(verbose_name="контрольный орган"), + ), + ] diff --git a/src/apps/external_data/models.py b/src/apps/external_data/models.py index d78655d..090dafb 100644 --- a/src/apps/external_data/models.py +++ b/src/apps/external_data/models.py @@ -44,7 +44,12 @@ class IndustrialCertificate(UUIDPrimaryKeyMixin, TimestampMixin, models.Model): certificate_number = models.CharField( _("номер сертификата"), max_length=100, db_index=True ) - issue_date = models.DateField(_("дата выдачи"), null=True, blank=True) + issue_date = models.DateField( + _("дата выдачи"), + null=True, + blank=True, + db_index=True, + ) expiry_date = models.DateField(_("дата окончания"), null=True, blank=True) certificate_file_url = models.TextField( _("ссылка на файл сертификата"), blank=True, default="" @@ -96,7 +101,7 @@ class ProsecutorCheck(UUIDPrimaryKeyMixin, TimestampMixin, models.Model): _("регистрационный номер"), max_length=64, db_index=True ) law_type = models.CharField(_("тип закона"), max_length=32, db_index=True) - control_authority = models.CharField(_("контрольный орган"), max_length=255) + control_authority = models.TextField(_("контрольный орган")) prosecutor_office = models.CharField( _("прокуратура"), max_length=255, blank=True, default="" ) @@ -154,6 +159,13 @@ class ArbitrationCase(UUIDPrimaryKeyMixin, TimestampMixin, models.Model): court_name = models.CharField(_("суд"), max_length=255) party_role = models.CharField(_("роль стороны"), max_length=64, db_index=True) status = models.CharField(_("статус"), max_length=64, db_index=True) + claim_amount = models.DecimalField( + _("размер иска"), + max_digits=20, + decimal_places=2, + null=True, + blank=True, + ) decision_date = models.DateField(_("дата решения"), db_index=True) class Meta: @@ -254,7 +266,12 @@ class InformationSecurityRegistryEntry( blank=True, default="", ) - issued_at = models.DateField(_("дата выдачи"), null=True, blank=True) + issued_at = models.DateField( + _("дата выдачи"), + null=True, + blank=True, + db_index=True, + ) expires_at = models.DateField(_("дата окончания"), null=True, blank=True) class Meta: diff --git a/src/apps/external_data/serializers.py b/src/apps/external_data/serializers.py index 26364d3..4711261 100644 --- a/src/apps/external_data/serializers.py +++ b/src/apps/external_data/serializers.py @@ -113,6 +113,7 @@ class ArbitrationCaseSerializer(serializers.ModelSerializer): "court_name", "party_role", "status", + "claim_amount", "decision_date", ] diff --git a/src/apps/external_data/source_record_export.py b/src/apps/external_data/source_record_export.py new file mode 100644 index 0000000..f9adc60 --- /dev/null +++ b/src/apps/external_data/source_record_export.py @@ -0,0 +1,1419 @@ +"""Prepared file exports for State Corp external source data.""" + +from __future__ import annotations + +import csv +import json +import os +import re +import secrets +import shutil +import zipfile +from collections.abc import Iterable, Iterator, Sequence +from dataclasses import dataclass +from datetime import UTC, date, datetime +from decimal import Decimal +from itertools import islice +from pathlib import Path +from tempfile import NamedTemporaryFile +from typing import Any, BinaryIO, cast +from uuid import UUID, uuid4 + +from apps.external_data.models import ( + ArbitrationCase, + BankruptcyProcedure, + DefenseUnreliableSupplier, + FinancialReport, + FinancialReportLine, + IndustrialCertificate, + IndustrialProduct, + InformationSecurityRegistryEntry, + LaborVacancy, + ManufacturerRegistryEntry, + ProsecutorCheck, + PublicProcurement, +) +from django.conf import settings +from django.core.cache import cache +from django.db.models import Prefetch, Q, QuerySet, prefetch_related_objects +from django.utils import timezone +from openpyxl import Workbook + +EXPORT_FORMAT_CSV = "csv" +EXPORT_FORMAT_XLSX = "xlsx" +EXPORT_FORMAT_JSON = "json" +EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON) +FINANCIAL_SOURCE_GROUP = "financial_indicators" +EXPORT_MANIFEST_VERSION = 2 +CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json" +GENERATION_MANIFEST_FILE_NAME = "manifest.json" +GENERATION_DIRECTORY_NAME = "generations" +SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE = 1000 +SOURCE_RECORD_EXPORT_ZIP_CHUNK_SIZE = 1024 * 1024 +EXCEL_MAX_DATA_ROWS_PER_SHEET = 1_048_575 +DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000 +DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60 +SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "external-data:source-record-exports:ticket" +SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}") +EXPORT_PROVIDER_URL_PATTERN = re.compile( + r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*", + flags=re.IGNORECASE, +) +EXPORT_PROVIDER_NAME_PATTERN = re.compile( + r"(? int: + return len( + { + (artifact.source_group, artifact.file_format) + for artifact in self.artifacts + } + ) + + @property + def files_count(self) -> int: + return len(self.artifacts) + + @property + def total_size(self) -> int: + return sum(artifact.size for artifact in self.artifacts) + + +@dataclass(frozen=True) +class SourceRecordExportArchive: + """A request-specific ZIP streamed only from prepared files.""" + + archive_name: str + archive_chunks: Iterable[bytes] + files_count: int + generated_at: str + + +@dataclass(frozen=True) +class SourceRecordExportDownloadTicket: + """Short-lived capability for one native browser download.""" + + ticket: str + archive_name: str + expires_in: int + + +class _StreamingZipSink: + """Unseekable zipfile target whose written chunks can be drained.""" + + def __init__(self) -> None: + self._offset = 0 + self._chunks: list[bytes] = [] + + def write(self, data: bytes) -> int: + rendered_data = bytes(data) + self._chunks.append(rendered_data) + self._offset += len(rendered_data) + return len(rendered_data) + + def tell(self) -> int: + return self._offset + + def flush(self) -> None: + return None + + def drain(self) -> tuple[bytes, ...]: + chunks = tuple(self._chunks) + self._chunks.clear() + return chunks + + +def build_source_record_export_artifacts( + *, + now: datetime | None = None, + export_directory: str | Path | None = None, +) -> SourceRecordExportGeneration: + """Build all files on disk and atomically publish a new generation.""" + + root_directory = _resolve_export_directory(export_directory) + generations_directory = root_directory / GENERATION_DIRECTORY_NAME + root_directory.mkdir(parents=True, exist_ok=True) + generations_directory.mkdir(parents=True, exist_ok=True) + + generated_at_datetime = _normalize_generation_datetime(now or timezone.now()) + export_year = _export_year(generated_at_datetime) + generation_id = ( + f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}" + ) + staging_directory = generations_directory / f".building-{generation_id}" + final_directory = generations_directory / generation_id + staging_directory.mkdir() + + try: + artifacts: list[SourceRecordExportArtifact] = [] + source_record_counts: dict[str, int] = {} + + for source_group, source_spec in SOURCE_GROUP_EXPORT_SPECS.items(): + row_spool_path = staging_directory / f".{source_group}.rows.json" + headers, records_count = _spool_source_group_rows( + source_spec=source_spec, + output_path=row_spool_path, + export_year=export_year, + ) + source_record_counts[source_group] = records_count + + try: + for file_format in _source_group_export_formats(source_group): + file_name = _build_source_group_file_name( + source_group=source_group, + file_format=file_format, + ) + artifact_paths = _render_source_group_artifact( + row_spool_path=row_spool_path, + output_path=staging_directory / file_name, + headers=headers, + file_format=file_format, + records_count=records_count, + ) + parts_count = len(artifact_paths) + artifacts.extend( + SourceRecordExportArtifact( + source_group=source_group, + file_format=file_format, + file_name=artifact_path.name, + path=final_directory / artifact_path.name, + size=artifact_path.stat().st_size, + records_count=records_count, + part_number=part_number, + parts_count=parts_count, + ) + for part_number, artifact_path in enumerate( + artifact_paths, + start=1, + ) + ) + finally: + row_spool_path.unlink(missing_ok=True) + + generation = SourceRecordExportGeneration( + generation_id=generation_id, + generated_at=generated_at_datetime.isoformat(), + export_year=export_year, + artifacts=tuple(artifacts), + records_count=sum(source_record_counts.values()), + ) + manifest_payload = _generation_manifest_payload( + generation, + root_directory=root_directory, + ) + _write_json_file( + staging_directory / GENERATION_MANIFEST_FILE_NAME, + manifest_payload, + ) + os.replace(staging_directory, final_directory) + _write_json_file_atomically( + root_directory / CURRENT_EXPORT_MANIFEST_FILE_NAME, + manifest_payload, + ) + _cleanup_stale_generations( + generations_directory=generations_directory, + current_generation_id=generation_id, + ) + return generation + except Exception: + if staging_directory.exists(): + shutil.rmtree(staging_directory) + raise + + +def load_current_source_record_export_generation( + *, + export_directory: str | Path | None = None, +) -> SourceRecordExportGeneration: + """Load and validate the atomically published current generation.""" + + root_directory = _resolve_export_directory(export_directory) + manifest_path = root_directory / CURRENT_EXPORT_MANIFEST_FILE_NAME + try: + manifest_payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError) as exc: + raise SourceRecordExportArtifactsUnavailable( + "Prepared source-record export is not available." + ) from exc + + return _generation_from_manifest( + manifest_payload, + root_directory=root_directory, + ) + + +def build_source_records_export_archive( + *, + source_groups: Sequence[str], + export_format: str, + export_directory: str | Path | None = None, + requested_at: datetime | None = None, +) -> SourceRecordExportArchive: + """Package selected prepared files without querying external-data tables.""" + + root_directory = _resolve_export_directory(export_directory) + generation = load_current_source_record_export_generation( + export_directory=root_directory, + ) + requested_at_datetime = _normalize_generation_datetime( + requested_at or timezone.now() + ) + if generation.export_year != _export_year(requested_at_datetime): + raise SourceRecordExportArtifactsUnavailable( + "Prepared source-record export belongs to a different calendar year." + ) + artifacts_by_key: dict[ + tuple[str, str], + list[SourceRecordExportArtifact], + ] = {} + for artifact in generation.artifacts: + artifacts_by_key.setdefault( + (artifact.source_group, artifact.file_format), + [], + ).append(artifact) + selected_artifacts: list[SourceRecordExportArtifact] = [] + + for source_group in source_groups: + file_format = _resolve_source_group_export_format( + source_group=source_group, + requested_format=export_format, + ) + artifacts = artifacts_by_key.get((source_group, file_format), []) + if not artifacts or any(not artifact.path.is_file() for artifact in artifacts): + raise SourceRecordExportArtifactsUnavailable( + f"Prepared export artifact is missing: {source_group}/{file_format}." + ) + selected_artifacts.extend( + sorted(artifacts, key=lambda artifact: artifact.part_number) + ) + + return SourceRecordExportArchive( + archive_name=_build_source_records_archive_name( + source_groups=source_groups, + requested_at=requested_at_datetime, + ), + archive_chunks=_stream_zip_archive(selected_artifacts), + files_count=len(selected_artifacts), + generated_at=generation.generated_at, + ) + + +def create_source_record_export_download_ticket( + *, + source_groups: Sequence[str], + export_format: str, +) -> SourceRecordExportDownloadTicket: + """Validate prepared files and cache a short-lived download capability.""" + + requested_at = _normalize_generation_datetime(timezone.now()) + package = build_source_records_export_archive( + source_groups=source_groups, + export_format=export_format, + requested_at=requested_at, + ) + expires_in = max( + 1, + int( + getattr( + settings, + "SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS", + DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS, + ) + ), + ) + payload = { + "sources": list(source_groups), + "format": export_format, + "requested_at": requested_at.isoformat(), + } + for _attempt in range(3): + ticket = secrets.token_urlsafe(32) + if cache.add( + _source_record_export_ticket_cache_key(ticket), + payload, + timeout=expires_in, + ): + return SourceRecordExportDownloadTicket( + ticket=ticket, + archive_name=package.archive_name, + expires_in=expires_in, + ) + raise RuntimeError("Could not allocate a source-record export download ticket.") + + +def consume_source_record_export_download_ticket( + ticket: str, +) -> SourceRecordExportArchive: + """Consume a download ticket before streaming the prepared archive.""" + + if not SOURCE_RECORD_EXPORT_TICKET_PATTERN.fullmatch(ticket): + raise SourceRecordExportTicketInvalid + + cache_key = _source_record_export_ticket_cache_key(ticket) + payload = cache.get(cache_key) + if payload is None: + raise SourceRecordExportTicketInvalid + cache.delete(cache_key) + + try: + source_groups = payload["sources"] + export_format = payload["format"] + requested_at_value = payload["requested_at"] + if ( + not isinstance(source_groups, list) + or not source_groups + or any( + not isinstance(source_group, str) + or source_group not in SOURCE_GROUP_EXPORT_SPECS + for source_group in source_groups + ) + or len(source_groups) != len(set(source_groups)) + or export_format not in EXPORT_FORMATS + or not isinstance(requested_at_value, str) + ): + raise ValueError + requested_at = datetime.fromisoformat(requested_at_value) + if timezone.is_naive(requested_at): + raise ValueError + except (KeyError, TypeError, ValueError): + raise SourceRecordExportTicketInvalid from None + + return build_source_records_export_archive( + source_groups=source_groups, + export_format=export_format, + requested_at=requested_at, + ) + + +def _source_record_export_ticket_cache_key(ticket: str) -> str: + return f"{SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX}:{ticket}" + + +def _stream_zip_archive( + artifacts: Sequence[SourceRecordExportArtifact], +) -> Iterator[bytes]: + sink = _StreamingZipSink() + with zipfile.ZipFile( + cast(BinaryIO, sink), + mode="w", + compression=zipfile.ZIP_STORED, + allowZip64=True, + ) as archive: + for artifact in artifacts: + with artifact.path.open("rb") as source_file: + with archive.open( + artifact.file_name, + mode="w", + force_zip64=True, + ) as archive_entry: + yield from sink.drain() + while chunk := source_file.read( + SOURCE_RECORD_EXPORT_ZIP_CHUNK_SIZE + ): + archive_entry.write(chunk) + yield from sink.drain() + yield from sink.drain() + yield from sink.drain() + + +def _resolve_export_directory(export_directory: str | Path | None) -> Path: + if export_directory is not None: + return Path(export_directory) + + configured_directory = getattr( + settings, + "SOURCE_RECORD_EXPORT_DIRECTORY", + Path(settings.MEDIA_ROOT) / "source-record-exports", + ) + return Path(str(configured_directory)) + + +def _normalize_generation_datetime(value: datetime) -> datetime: + if timezone.is_naive(value): + value = timezone.make_aware(value, UTC) + return value.astimezone(UTC) + + +def _export_year(value: datetime) -> int: + """Return the calendar year in the configured service timezone.""" + return timezone.localtime(value).year + + +def _build_source_records_archive_name( + *, + source_groups: Sequence[str], + requested_at: datetime, +) -> str: + source_name = "__".join( + SOURCE_GROUP_EXPORT_SPECS[source_group].file_stem + for source_group in source_groups + ) + timestamp = timezone.localtime(requested_at).strftime("%Y%m%d_%H%M%S") + return f"{source_name}_{timestamp}.zip" + + +def _source_group_export_formats(source_group: str) -> tuple[str, ...]: + if source_group == FINANCIAL_SOURCE_GROUP: + return (EXPORT_FORMAT_JSON,) + return EXPORT_FORMATS + + +def _resolve_source_group_export_format( + *, + source_group: str, + requested_format: str, +) -> str: + if source_group == FINANCIAL_SOURCE_GROUP: + return EXPORT_FORMAT_JSON + return requested_format + + +def _build_source_group_file_name(*, source_group: str, file_format: str) -> str: + return f"{SOURCE_GROUP_EXPORT_SPECS[source_group].file_stem}.{file_format}" + + +def _source_model_queryset( + model_spec: SourceModelExportSpec, + *, + export_year: int, +) -> QuerySet: + queryset = cast( + QuerySet, + model_spec.model.objects.select_related("organization").order_by(), + ) + if model_spec.export_year_lookup: + return queryset.filter( + **{model_spec.export_year_lookup: export_year} + ).distinct() + if model_spec.record_date_field: + return queryset.filter( + Q(**{f"{model_spec.record_date_field}__year": export_year}) + | Q( + **{ + f"{model_spec.record_date_field}__isnull": True, + "created_at__year": export_year, + } + ) + ) + return queryset.filter(created_at__year=export_year) + + +def _iter_source_model_records( + model_spec: SourceModelExportSpec, + *, + export_year: int, +) -> Iterator[Any]: + iterator = _source_model_queryset( + model_spec, + export_year=export_year, + ).iterator(chunk_size=SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE) + while True: + batch = list(islice(iterator, SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE)) + if not batch: + return + if model_spec.prefetch_related: + prefetches = [ + Prefetch( + related_name, + queryset=FinancialReportLine.objects.filter( + year=export_year + ).order_by(), + ) + if related_name == "lines" + and model_spec.export_year_lookup == "lines__year" + else related_name + for related_name in model_spec.prefetch_related + ] + prefetch_related_objects(batch, *prefetches) + yield from batch + + +def _source_group_headers(source_spec: SourceGroupExportSpec) -> list[str]: + payload_fields: list[str] = [] + for model_spec in source_spec.models: + for field_name in _payload_export_field_names(model_spec): + payload_field_name = f"payload.{field_name}" + if payload_field_name not in payload_fields: + payload_fields.append(payload_field_name) + payload_fields.sort() + if source_spec.source_group == FINANCIAL_SOURCE_GROUP: + payload_fields.append("financial_lines") + return [ + *ORGANIZATION_EXPORT_FIELDS, + *SOURCE_RECORD_EXPORT_FIELDS, + *payload_fields, + ] + + +def _spool_source_group_rows( + *, + source_spec: SourceGroupExportSpec, + output_path: Path, + export_year: int, +) -> tuple[list[str], int]: + records_count = 0 + with output_path.open("w", encoding="utf-8", newline="") as output: + output.write("[") + is_first_row = True + for model_spec in source_spec.models: + for record in _iter_source_model_records( + model_spec, + export_year=export_year, + ): + row = _build_record_row( + record, + source_spec=source_spec, + model_spec=model_spec, + ) + if is_first_row: + output.write("\n") + is_first_row = False + else: + output.write(",\n") + output.write(json.dumps(row, ensure_ascii=False, separators=(",", ":"))) + records_count += 1 + if not is_first_row: + output.write("\n") + output.write("]") + return _source_group_headers(source_spec), records_count + + +def _render_source_group_artifact( + *, + row_spool_path: Path, + output_path: Path, + headers: Sequence[str], + file_format: str, + records_count: int, +) -> tuple[Path, ...]: + if file_format == EXPORT_FORMAT_CSV: + _render_csv_file( + row_spool_path=row_spool_path, + output_path=output_path, + headers=headers, + ) + return (output_path,) + if file_format == EXPORT_FORMAT_XLSX: + return _render_xlsx_files( + row_spool_path=row_spool_path, + output_path=output_path, + headers=headers, + records_count=records_count, + ) + os.link(row_spool_path, output_path) + return (output_path,) + + +def _iter_spooled_rows(row_spool_path: Path) -> Iterator[dict[str, Any]]: + with row_spool_path.open("r", encoding="utf-8") as rows_file: + for line in rows_file: + serialized_row = line.strip() + if not serialized_row or serialized_row in {"[", "]", "[]"}: + continue + if serialized_row.endswith(","): + serialized_row = serialized_row[:-1] + yield json.loads(serialized_row) + + +def _render_csv_file( + *, + row_spool_path: Path, + output_path: Path, + headers: Sequence[str], +) -> None: + with output_path.open("w", encoding="utf-8-sig", newline="") as output: + writer = csv.DictWriter(output, fieldnames=list(headers), lineterminator="\n") + writer.writeheader() + for row in _iter_spooled_rows(row_spool_path): + writer.writerow( + {key: _serialize_flat_value(row.get(key)) for key in headers} + ) + + +def _render_xlsx_files( + *, + row_spool_path: Path, + output_path: Path, + headers: Sequence[str], + records_count: int, +) -> tuple[Path, ...]: + rows_per_file = min( + EXCEL_MAX_DATA_ROWS_PER_SHEET, + max( + 1, + int( + getattr( + settings, + "SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE", + DEFAULT_XLSX_DATA_ROWS_PER_FILE, + ) + ), + ), + ) + parts_count = max(1, (records_count + rows_per_file - 1) // rows_per_file) + part_paths = tuple( + _build_xlsx_part_path( + output_path=output_path, + part_number=part_number, + parts_count=parts_count, + ) + for part_number in range(1, parts_count + 1) + ) + part_number = 1 + rows_in_file = 0 + workbook, worksheet = _new_export_workbook(headers) + + for row in _iter_spooled_rows(row_spool_path): + if rows_in_file >= rows_per_file: + workbook.save(part_paths[part_number - 1]) + workbook.close() + part_number += 1 + rows_in_file = 0 + workbook, worksheet = _new_export_workbook(headers) + worksheet.append([_serialize_flat_value(row.get(key)) for key in headers]) + rows_in_file += 1 + + workbook.save(part_paths[part_number - 1]) + workbook.close() + if part_number != parts_count: + raise ValueError("Unexpected XLSX source-record export parts count.") + return part_paths + + +def _new_export_workbook(headers: Sequence[str]): + workbook = Workbook(write_only=True) + worksheet = workbook.create_sheet(title="data") + worksheet.append(list(headers)) + return workbook, worksheet + + +def _build_xlsx_part_path( + *, + output_path: Path, + part_number: int, + parts_count: int, +) -> Path: + if parts_count == 1: + return output_path + return output_path.with_name( + f"{output_path.stem}-part-{part_number:03d}{output_path.suffix}" + ) + + +def _build_record_row( + record: Any, + *, + source_spec: SourceGroupExportSpec, + model_spec: SourceModelExportSpec, +) -> dict[str, Any]: + organization = record.organization + row: dict[str, Any] = { + "Наименование": organization.full_name + or organization.short_name + or organization.name, + "ИНН": organization.inn, + "ОГРН": organization.ogrn, + "КПП": organization.kpp, + "ОКПО": organization.okpo, + "uid": record.id, + "source_group": source_spec.source_group, + "source": _source_value(record, model_spec=model_spec), + "record_type": model_spec.record_type, + "external_id": _record_field_value(record, model_spec.external_id_field), + "title": _title_value(record, model_spec=model_spec), + "record_date": _record_field_value(record, model_spec.record_date_field), + "amount": _record_field_value(record, model_spec.amount_field), + "status": _record_field_value(record, model_spec.status_field), + "url": _record_field_value(record, model_spec.url_field), + "load_batch": _record_field_value(record, model_spec.load_batch_field), + "created_at": record.created_at, + "updated_at": record.updated_at, + **{ + f"payload.{field_name}": _payload_field_value( + record, + organization=organization, + model_spec=model_spec, + field_name=field_name, + ) + for field_name in _payload_export_field_names(model_spec) + }, + } + if source_spec.source_group == FINANCIAL_SOURCE_GROUP: + row["financial_lines"] = [ + { + field_name: getattr(line, field_name) + for field_name in FINANCIAL_LINE_FIELDS + } + for line in record.lines.all() + ] + public_row = _sanitize_export_provider_value(row) + return {key: _serialize_json_value(value) for key, value in public_row.items()} + + +def _source_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any: + source_value = _record_field_value(record, model_spec.source_field) + return source_value or model_spec.source + + +def _title_value(record: Any, *, model_spec: SourceModelExportSpec) -> Any: + title_value = _record_field_value(record, model_spec.title_field) + if not title_value or not model_spec.title_prefix: + return title_value + return f"{model_spec.title_prefix}{title_value}" + + +def _payload_field_name(model_spec: SourceModelExportSpec, field_name: str) -> str: + return dict(model_spec.payload_aliases).get(field_name, field_name) + + +def _payload_export_field_names(model_spec: SourceModelExportSpec) -> tuple[str, ...]: + if model_spec.payload_export_fields is not None: + return model_spec.payload_export_fields + + field_names = ["inn", "ogrn", "organisation_name", "source"] + for field_name in model_spec.fields: + payload_field_name = _payload_field_name(model_spec, field_name) + if payload_field_name not in field_names: + field_names.append(payload_field_name) + return tuple(field_names) + + +def _payload_field_value( + record: Any, + *, + organization: Any, + model_spec: SourceModelExportSpec, + field_name: str, +) -> Any: + if field_name == "inn": + return organization.inn + if field_name == "ogrn": + return organization.ogrn + if field_name == "organisation_name": + return organization.full_name or organization.short_name or organization.name + if field_name == "source": + return _source_value(record, model_spec=model_spec) + + source_field_by_alias = { + alias: source_field for source_field, alias in model_spec.payload_aliases + } + source_field = source_field_by_alias.get(field_name, field_name) + return getattr(record, source_field, None) + + +def _record_field_value(record: Any, field_name: str | None) -> Any: + return getattr(record, field_name) if field_name else None + + +def _sanitize_export_provider_value(value: Any) -> Any: + """Remove provider wording from files without changing stored source data.""" + if isinstance(value, str): + sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value) + return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized) + if isinstance(value, dict): + return { + _sanitize_export_provider_value(key): _sanitize_export_provider_value(item) + for key, item in value.items() + } + if isinstance(value, list | tuple): + return [_sanitize_export_provider_value(item) for item in value] + return value + + +def _serialize_flat_value(value: Any) -> str | int | float | bool: + if value is None: + return "" + if isinstance(value, dict | list | tuple): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + if isinstance(value, str | int | float | bool): + return value + return str(value) + + +def _serialize_json_value(value: Any) -> Any: + if isinstance(value, dict): + return {key: _serialize_json_value(item) for key, item in value.items()} + if isinstance(value, list | tuple): + return [_serialize_json_value(item) for item in value] + if isinstance(value, Decimal | UUID): + return str(value) + if isinstance(value, datetime | date): + return value.isoformat() + return value + + +def _generation_manifest_payload( + generation: SourceRecordExportGeneration, + *, + root_directory: Path, +) -> dict[str, Any]: + return { + "version": EXPORT_MANIFEST_VERSION, + "generation_id": generation.generation_id, + "generated_at": generation.generated_at, + "export_year": generation.export_year, + "records_count": generation.records_count, + "artifacts_count": generation.artifacts_count, + "files_count": generation.files_count, + "total_size": generation.total_size, + "artifacts": [ + { + "source_group": artifact.source_group, + "format": artifact.file_format, + "file_name": artifact.file_name, + "relative_path": str(artifact.path.relative_to(root_directory)), + "size": artifact.size, + "records_count": artifact.records_count, + "part_number": artifact.part_number, + "parts_count": artifact.parts_count, + } + for artifact in generation.artifacts + ], + } + + +def _generation_from_manifest( + payload: Any, + *, + root_directory: Path, +) -> SourceRecordExportGeneration: + try: + if payload["version"] != EXPORT_MANIFEST_VERSION: + raise ValueError("Unsupported source-record export manifest version.") + generation_id = str(payload["generation_id"]) + generated_at = str(payload["generated_at"]) + datetime.fromisoformat(generated_at) + export_year = int(payload["export_year"]) + if not 1 <= export_year <= 9999: + raise ValueError("Source-record export year is invalid.") + records_count = int(payload["records_count"]) + artifact_payloads = payload["artifacts"] + if not isinstance(artifact_payloads, list): + raise TypeError("Manifest artifacts must be a list.") + + root_resolved = root_directory.resolve() + artifacts: list[SourceRecordExportArtifact] = [] + for artifact_payload in artifact_payloads: + artifact_path = root_directory / str(artifact_payload["relative_path"]) + artifact_path.resolve().relative_to(root_resolved) + if not artifact_path.is_file(): + raise FileNotFoundError(artifact_path) + source_group = str(artifact_payload["source_group"]) + file_format = str(artifact_payload["format"]) + file_name = str(artifact_payload["file_name"]) + artifact_size = int(artifact_payload["size"]) + part_number = int(artifact_payload.get("part_number", 1)) + parts_count = int(artifact_payload.get("parts_count", 1)) + expected_file_name = _build_source_group_file_name( + source_group=source_group, + file_format=file_format, + ) + if file_format == EXPORT_FORMAT_XLSX: + expected_file_name = _build_xlsx_part_path( + output_path=Path(expected_file_name), + part_number=part_number, + parts_count=parts_count, + ).name + if file_name != expected_file_name: + raise ValueError("Unexpected source-record export artifact name.") + if artifact_size != artifact_path.stat().st_size: + raise ValueError("Source-record export artifact size mismatch.") + artifacts.append( + SourceRecordExportArtifact( + source_group=source_group, + file_format=file_format, + file_name=file_name, + path=artifact_path, + size=artifact_size, + records_count=int(artifact_payload["records_count"]), + part_number=part_number, + parts_count=parts_count, + ) + ) + + _validate_manifest_artifact_parts(artifacts) + except (KeyError, TypeError, ValueError, OSError) as exc: + raise SourceRecordExportArtifactsUnavailable( + "Prepared source-record export manifest is invalid." + ) from exc + + return SourceRecordExportGeneration( + generation_id=generation_id, + generated_at=generated_at, + export_year=export_year, + artifacts=tuple(artifacts), + records_count=records_count, + ) + + +def _validate_manifest_artifact_parts( + artifacts: Sequence[SourceRecordExportArtifact], +) -> None: + expected_artifact_keys = { + (source_group, file_format) + for source_group in SOURCE_GROUP_EXPORT_SPECS + for file_format in _source_group_export_formats(source_group) + } + artifacts_by_key: dict[ + tuple[str, str], + list[SourceRecordExportArtifact], + ] = {} + for artifact in artifacts: + artifacts_by_key.setdefault( + (artifact.source_group, artifact.file_format), + [], + ).append(artifact) + if set(artifacts_by_key) != expected_artifact_keys: + raise ValueError("Source-record export manifest matrix is incomplete.") + + for artifact_key, artifact_parts in artifacts_by_key.items(): + parts_count = len(artifact_parts) + if ( + {artifact.parts_count for artifact in artifact_parts} != {parts_count} + or {artifact.part_number for artifact in artifact_parts} + != set(range(1, parts_count + 1)) + or (artifact_key[1] != EXPORT_FORMAT_XLSX and parts_count != 1) + ): + raise ValueError("Source-record export artifact parts are invalid.") + + +def _write_json_file(file_path: Path, payload: dict[str, Any]) -> None: + file_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + +def _write_json_file_atomically(file_path: Path, payload: dict[str, Any]) -> None: + temp_path: Path | None = None + try: + with NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=file_path.parent, + prefix=f".{file_path.name}.", + suffix=".tmp", + delete=False, + ) as temp_file: + json.dump(payload, temp_file, ensure_ascii=False, indent=2) + temp_file.flush() + os.fsync(temp_file.fileno()) + temp_path = Path(temp_file.name) + os.replace(temp_path, file_path) + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) + + +def _cleanup_stale_generations( + *, + generations_directory: Path, + current_generation_id: str, +) -> None: + generations_to_keep = max( + 1, + int(getattr(settings, "SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP", 2)), + ) + published_generations = sorted( + ( + path + for path in generations_directory.iterdir() + if path.is_dir() + and not path.name.startswith(".") + and (path / GENERATION_MANIFEST_FILE_NAME).is_file() + ), + key=lambda path: path.name, + reverse=True, + ) + retained_names = {current_generation_id} + retained_names.update( + path.name for path in published_generations[:generations_to_keep] + ) + + for generation_directory in published_generations: + if generation_directory.name not in retained_names: + shutil.rmtree(generation_directory) diff --git a/src/apps/external_data/tasks.py b/src/apps/external_data/tasks.py new file mode 100644 index 0000000..8950260 --- /dev/null +++ b/src/apps/external_data/tasks.py @@ -0,0 +1,47 @@ +"""Celery tasks for prepared external-data exports.""" + +import logging + +from apps.core.tasks import PeriodicTask as CorePeriodicTask +from apps.external_data.source_record_export import ( + build_source_record_export_artifacts, +) +from celery import shared_task +from django.conf import settings +from django.core.cache import cache + +logger = logging.getLogger(__name__) + + +@shared_task(bind=True, base=CorePeriodicTask) +def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 + """Build and atomically publish the nightly external-data export matrix.""" + + lock_key = getattr( + settings, + "SOURCE_RECORD_EXPORT_LOCK_KEY", + "external-data:source-record-exports:lock", + ) + lock_ttl = int( + getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60) + ) + if not cache.add(lock_key, "1", timeout=lock_ttl): + logger.info("Source-record export generation skipped: lock is already held") + return {"status": "skipped", "reason": "locked"} + + try: + generation = build_source_record_export_artifacts() + result = { + "status": "success", + "generation_id": generation.generation_id, + "generated_at": generation.generated_at, + "export_year": generation.export_year, + "artifacts_count": generation.artifacts_count, + "files_count": generation.files_count, + "records_count": generation.records_count, + "total_size": generation.total_size, + } + logger.info("Source-record export generation published: %s", result) + return result + finally: + cache.delete(lock_key) diff --git a/src/apps/form_3/migrations/0003_reporting_analytics_fields.py b/src/apps/form_3/migrations/0003_reporting_analytics_fields.py new file mode 100644 index 0000000..ae28cf6 --- /dev/null +++ b/src/apps/form_3/migrations/0003_reporting_analytics_fields.py @@ -0,0 +1,73 @@ +# Generated by Django 3.2.25 on 2026-07-28 08:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('form_3', '0002_auto_20260328_1621'), + ] + + operations = [ + migrations.AddField( + model_name='formf3record', + name='average_employee_age', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='средний возраст сотрудников'), + ), + migrations.AddField( + model_name='formf3record', + name='employees_20_29', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='работники 20-29 лет'), + ), + migrations.AddField( + model_name='formf3record', + name='employees_30_39', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='работники 30-39 лет'), + ), + migrations.AddField( + model_name='formf3record', + name='employees_40_49', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='работники 40-49 лет'), + ), + migrations.AddField( + model_name='formf3record', + name='employees_50_59', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='работники 50-59 лет'), + ), + migrations.AddField( + model_name='formf3record', + name='employees_over_60', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='работники старше 60 лет'), + ), + migrations.AddField( + model_name='formf3record', + name='employees_under_20', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='работники до 20 лет'), + ), + migrations.AddField( + model_name='formf3record', + name='machine_tools_and_equipment', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='станочный парк и оборудование, всего'), + ), + migrations.AddField( + model_name='formf3record', + name='military_utilization_rate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='уровень загрузки мощностей военного производства'), + ), + migrations.AddField( + model_name='formf3record', + name='shipped_goods_innovative', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=18, null=True, verbose_name='инновационные отгруженные товары, работы и услуги'), + ), + migrations.AddField( + model_name='formf3record', + name='shipped_goods_total', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=18, null=True, verbose_name='отгруженные товары, работы и услуги, всего'), + ), + migrations.AddField( + model_name='formf3record', + name='total_fixed_assets', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=18, null=True, verbose_name='основные фонды, всего'), + ), + ] diff --git a/src/apps/form_3/models.py b/src/apps/form_3/models.py index c7d7b27..1b972f6 100644 --- a/src/apps/form_3/models.py +++ b/src/apps/form_3/models.py @@ -67,8 +67,71 @@ class FormF3Record(ReportingPeriodMixin, TimestampMixin, models.Model): null=True, blank=True, ) + average_employee_age = models.DecimalField( + _("средний возраст сотрудников"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + employees_under_20 = models.PositiveIntegerField( + _("работники до 20 лет"), + null=True, + blank=True, + ) + employees_20_29 = models.PositiveIntegerField( + _("работники 20-29 лет"), + null=True, + blank=True, + ) + employees_30_39 = models.PositiveIntegerField( + _("работники 30-39 лет"), + null=True, + blank=True, + ) + employees_40_49 = models.PositiveIntegerField( + _("работники 40-49 лет"), + null=True, + blank=True, + ) + employees_50_59 = models.PositiveIntegerField( + _("работники 50-59 лет"), + null=True, + blank=True, + ) + employees_over_60 = models.PositiveIntegerField( + _("работники старше 60 лет"), + null=True, + blank=True, + ) # === Оборудование - общие данные === + shipped_goods_total = models.DecimalField( + _("отгруженные товары, работы и услуги, всего"), + max_digits=18, + decimal_places=2, + null=True, + blank=True, + ) + shipped_goods_innovative = models.DecimalField( + _("инновационные отгруженные товары, работы и услуги"), + max_digits=18, + decimal_places=2, + null=True, + blank=True, + ) + total_fixed_assets = models.DecimalField( + _("основные фонды, всего"), + max_digits=18, + decimal_places=2, + null=True, + blank=True, + ) + machine_tools_and_equipment = models.PositiveIntegerField( + _("станочный парк и оборудование, всего"), + null=True, + blank=True, + ) total_equipment = models.PositiveIntegerField( _("всего оборудования"), null=True, @@ -127,6 +190,13 @@ class FormF3Record(ReportingPeriodMixin, TimestampMixin, models.Model): null=True, blank=True, ) + military_utilization_rate = models.DecimalField( + _("уровень загрузки мощностей военного производства"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) avg_shift_work = models.DecimalField( _("средняя сменность работы"), max_digits=5, diff --git a/src/apps/form_3/services.py b/src/apps/form_3/services.py index 386266b..1d496c5 100644 --- a/src/apps/form_3/services.py +++ b/src/apps/form_3/services.py @@ -97,81 +97,118 @@ class FormF3Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF3Record]): OKPO_COLUMN = 1 OGRN_COLUMN = 2 INN_COLUMN = 3 + DATA_START_ROW = 3 def get_column_mappings(self) -> list[ColumnMapping]: """Маппинг колонок Excel на поля модели.""" return [ - # Кадры ColumnMapping( 4, - "Средняя численность работников", - "avg_employees", + "Отгруженные товары собственного производства", + "shipped_goods_total", field_type="decimal", ), ColumnMapping( 5, - "Производственный персонал", - "production_workers", + "из них инновационные товары", + "shipped_goods_innovative", field_type="decimal", ), ColumnMapping( 6, - "Инженерно-технические работники", - "engineering_workers", + "Всего основных фондов", + "total_fixed_assets", field_type="decimal", ), ColumnMapping( 7, - "Административный персонал", - "administrative_workers", + "Станочный парк и оборудование, всего", + "machine_tools_and_equipment", + field_type="int", + ), + ColumnMapping( + 8, + "Станочный парк и оборудование - импортное оборудование", + "imported_equipment", + field_type="int", + ), + ColumnMapping( + 9, + "Оборудование с возрастом от 5 до 10 лет", + "equipment_age_5_10", + field_type="int", + ), + ColumnMapping( + 10, + "Оборудование с возрастом до 5 лет", + "equipment_age_under_5", + field_type="int", + ), + ColumnMapping( + 11, + "Фактический (физический) износ оборудования", + "physical_wear_percent", field_type="decimal", ), - # Оборудование - общие данные - ColumnMapping(8, "Всего оборудования", "total_equipment", field_type="int"), ColumnMapping( - 9, "Отечественное оборудование", "domestic_equipment", field_type="int" + 12, + "Средний уровень загрузки производственных мощностей (всего)", + "utilization_rate", + field_type="decimal", ), ColumnMapping( - 10, "Импортное оборудование", "imported_equipment", field_type="int" - ), - # Оборудование по возрасту - ColumnMapping( - 11, "Оборудование до 5 лет", "equipment_age_under_5", field_type="int" + 13, + "Средний уровень загрузки производственных мощностей военного производства", + "military_utilization_rate", + field_type="decimal", ), ColumnMapping( - 12, "Оборудование 5-10 лет", "equipment_age_5_10", field_type="int" - ), - ColumnMapping( - 13, "Оборудование 10-15 лет", "equipment_age_10_15", field_type="int" - ), - ColumnMapping( - 14, "Оборудование 15-20 лет", "equipment_age_15_20", field_type="int" + 14, + "Средний возраст сотрудников", + "average_employee_age", + field_type="decimal", ), ColumnMapping( 15, - "Оборудование свыше 20 лет", - "equipment_age_over_20", + "Количество работников предприятия до 20 лет", + "employees_under_20", field_type="int", ), - # Износ и использование ColumnMapping( - 16, "Физический износ, %", "physical_wear_percent", field_type="decimal" + 16, + "Количество работников предприятия 20-29 лет", + "employees_20_29", + field_type="int", ), ColumnMapping( - 17, "Коэффициент загрузки", "utilization_rate", field_type="decimal" + 17, + "Количество работников предприятия 30-39 лет", + "employees_30_39", + field_type="int", ), ColumnMapping( - 18, "Средняя сменность работы", "avg_shift_work", field_type="decimal" - ), - # Потребности - ColumnMapping( - 19, "Потребность в оборудовании", "equipment_needed", field_type="int" + 18, + "Количество работников предприятия 40-49 лет", + "employees_40_49", + field_type="int", ), ColumnMapping( - 20, "Потребность в кадрах", "workers_needed", field_type="int" + 19, + "Количество работников предприятия 50-59 лет", + "employees_50_59", + field_type="int", + ), + ColumnMapping( + 20, + "Количество работников предприятия старше 60 лет", + "employees_over_60", + field_type="int", ), ] + def validate_workbook(self) -> None: + self._validate_mapped_headers(header_rows=(1,)) + def get_next_batch_id(self) -> int: """Получить следующий номер загрузки.""" return FormF3Service.get_next_batch_id() diff --git a/src/apps/form_4/services.py b/src/apps/form_4/services.py index 42c5a7f..f5cf420 100644 --- a/src/apps/form_4/services.py +++ b/src/apps/form_4/services.py @@ -75,86 +75,63 @@ class FormF4Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF4Record]): OKPO_COLUMN = 1 OGRN_COLUMN = 2 INN_COLUMN = 3 + DATA_START_ROW = 3 def get_column_mappings(self) -> list[ColumnMapping]: return [ - # Выручка - ColumnMapping(4, "Выручка (РСБУ)", "revenue_rsbu", field_type="decimal"), - ColumnMapping(5, "Выручка (МСФО)", "revenue_ifrs", field_type="decimal"), + ColumnMapping( + 4, + "Выручка, в соответствии с РСБУ", + "revenue_rsbu", + field_type="decimal", + ), + ColumnMapping( + 5, + "Выручка, в соответствии с МСФО", + "revenue_ifrs", + field_type="decimal", + ), ColumnMapping( 6, - "Выручка прошлого года (РСБУ)", - "revenue_prev_rsbu", + "Чистая прибыль (убыток), в соответствии с РСБУ", + "net_profit_rsbu", field_type="decimal", ), ColumnMapping( 7, - "Выручка прошлого года (МСФО)", - "revenue_prev_ifrs", + "Чистая прибыль (убыток), в соответствии с МСФО", + "net_profit_ifrs", field_type="decimal", ), - # Прибыль ColumnMapping( - 8, "Чистая прибыль (РСБУ)", "net_profit_rsbu", field_type="decimal" + 8, + "Кредиты и займы, в соответствии с РСБУ", + "loans_rsbu", + field_type="decimal", ), ColumnMapping( - 9, "Чистая прибыль (МСФО)", "net_profit_ifrs", field_type="decimal" + 9, + "Кредиты и займы, в соответствии с МСФО", + "loans_ifrs", + field_type="decimal", ), ColumnMapping( - 10, "Валовая прибыль (РСБУ)", "gross_profit_rsbu", field_type="decimal" + 10, + "EBITDA, в соответствии с РСБУ", + "ebitda_rsbu", + field_type="decimal", ), ColumnMapping( 11, - "Операционная прибыль (РСБУ)", - "operating_profit_rsbu", + "EBITDA, в соответствии с МСФО", + "ebitda_ifrs", field_type="decimal", ), - # EBITDA - ColumnMapping(12, "EBITDA (РСБУ)", "ebitda_rsbu", field_type="decimal"), - ColumnMapping(13, "EBITDA (МСФО)", "ebitda_ifrs", field_type="decimal"), - # Долговая нагрузка - ColumnMapping( - 14, "Кредиты и займы (РСБУ)", "loans_rsbu", field_type="decimal" - ), - ColumnMapping( - 15, "Кредиты и займы (МСФО)", "loans_ifrs", field_type="decimal" - ), - ColumnMapping( - 16, "Чистый долг (РСБУ)", "net_debt_rsbu", field_type="decimal" - ), - ColumnMapping( - 17, "Чистый долг (МСФО)", "net_debt_ifrs", field_type="decimal" - ), - ColumnMapping(18, "Долг/EBITDA", "debt_to_ebitda", field_type="decimal"), - # Активы и капитал - ColumnMapping( - 19, "Активы (РСБУ)", "total_assets_rsbu", field_type="decimal" - ), - ColumnMapping( - 20, "Активы (МСФО)", "total_assets_ifrs", field_type="decimal" - ), - ColumnMapping( - 21, "Собственный капитал (РСБУ)", "equity_rsbu", field_type="decimal" - ), - ColumnMapping( - 22, "Собственный капитал (МСФО)", "equity_ifrs", field_type="decimal" - ), - # Рентабельность - ColumnMapping(23, "ROE", "roe", field_type="decimal"), - ColumnMapping(24, "ROA", "roa", field_type="decimal"), - ColumnMapping(25, "ROS", "ros", field_type="decimal"), - # Инвестиции - ColumnMapping(26, "CAPEX", "capex", field_type="decimal"), - ColumnMapping(27, "Затраты на НИОКР", "rd_expenses", field_type="decimal"), - # Дивиденды - ColumnMapping( - 28, "Выплаченные дивиденды", "dividends_paid", field_type="decimal" - ), - ColumnMapping( - 29, "Дивидендная доходность", "dividend_yield", field_type="decimal" - ), ] + def validate_workbook(self) -> None: + self._validate_mapped_headers(header_rows=(1,)) + def get_next_batch_id(self) -> int: return FormF4Service.get_next_batch_id() diff --git a/src/apps/form_6/migrations/0003_reporting_analytics_fields.py b/src/apps/form_6/migrations/0003_reporting_analytics_fields.py new file mode 100644 index 0000000..213bc32 --- /dev/null +++ b/src/apps/form_6/migrations/0003_reporting_analytics_fields.py @@ -0,0 +1,128 @@ +# Generated by Django 3.2.25 on 2026-07-28 08:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('form_6', '0002_auto_20260328_1621'), + ] + + operations = [ + migrations.AddField( + model_name='formf6record', + name='age_10_15_imported', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='импортное оборудование 10-15 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_10_15_lease_share_itn_percent', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='доля времени работы для ПВН 10-15 лет, %'), + ), + migrations.AddField( + model_name='formf6record', + name='age_10_15_utilization_rate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='использование оборудования 10-15 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_10_15_weighted_load_factor', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='средневзвешенный коэффициент загрузки 10-15 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_15_20_imported', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='импортное оборудование 15-20 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_15_20_lease_share_itn_percent', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='доля времени работы для ПВН 15-20 лет, %'), + ), + migrations.AddField( + model_name='formf6record', + name='age_15_20_utilization_rate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='использование оборудования 15-20 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_15_20_weighted_load_factor', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='средневзвешенный коэффициент загрузки 15-20 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_5_10_imported', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='импортное оборудование 5-10 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_5_10_lease_share_itn_percent', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='доля времени работы для ПВН 5-10 лет, %'), + ), + migrations.AddField( + model_name='formf6record', + name='age_5_10_utilization_rate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='использование оборудования 5-10 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_5_10_weighted_load_factor', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='средневзвешенный коэффициент загрузки 5-10 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_over_20_imported', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='импортное оборудование свыше 20 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_over_20_lease_share_itn_percent', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='доля времени работы для ПВН свыше 20 лет, %'), + ), + migrations.AddField( + model_name='formf6record', + name='age_over_20_utilization_rate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='использование оборудования свыше 20 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_over_20_weighted_load_factor', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='средневзвешенный коэффициент загрузки свыше 20 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_under_5_imported', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='импортное оборудование до 5 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_under_5_lease_share_itn_percent', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='доля времени работы для ПВН до 5 лет, %'), + ), + migrations.AddField( + model_name='formf6record', + name='age_under_5_utilization_rate', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='использование оборудования до 5 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='age_under_5_weighted_load_factor', + field=models.DecimalField(blank=True, decimal_places=2, max_digits=5, null=True, verbose_name='средневзвешенный коэффициент загрузки до 5 лет'), + ), + migrations.AddField( + model_name='formf6record', + name='commissioned_equipment', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='введено в эксплуатацию'), + ), + migrations.AddField( + model_name='formf6record', + name='decommissioned_equipment', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='выведено из эксплуатации'), + ), + migrations.AddField( + model_name='formf6record', + name='notes', + field=models.TextField(blank=True, default='', verbose_name='примечание'), + ), + ] diff --git a/src/apps/form_6/models.py b/src/apps/form_6/models.py index 0c8e10e..16b48aa 100644 --- a/src/apps/form_6/models.py +++ b/src/apps/form_6/models.py @@ -68,6 +68,16 @@ class FormF6Record(ReportingPeriodMixin, TimestampMixin, models.Model): null=True, blank=True, ) + commissioned_equipment = models.PositiveIntegerField( + _("введено в эксплуатацию"), + null=True, + blank=True, + ) + decommissioned_equipment = models.PositiveIntegerField( + _("выведено из эксплуатации"), + null=True, + blank=True, + ) # === Возрастная структура === age_under_5 = models.PositiveIntegerField( @@ -95,6 +105,137 @@ class FormF6Record(ReportingPeriodMixin, TimestampMixin, models.Model): null=True, blank=True, ) + age_under_5_imported = models.PositiveIntegerField( + _("импортное оборудование до 5 лет"), + null=True, + blank=True, + ) + age_5_10_imported = models.PositiveIntegerField( + _("импортное оборудование 5-10 лет"), + null=True, + blank=True, + ) + age_10_15_imported = models.PositiveIntegerField( + _("импортное оборудование 10-15 лет"), + null=True, + blank=True, + ) + age_15_20_imported = models.PositiveIntegerField( + _("импортное оборудование 15-20 лет"), + null=True, + blank=True, + ) + age_over_20_imported = models.PositiveIntegerField( + _("импортное оборудование свыше 20 лет"), + null=True, + blank=True, + ) + + age_under_5_utilization_rate = models.DecimalField( + _("использование оборудования до 5 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_5_10_utilization_rate = models.DecimalField( + _("использование оборудования 5-10 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_10_15_utilization_rate = models.DecimalField( + _("использование оборудования 10-15 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_15_20_utilization_rate = models.DecimalField( + _("использование оборудования 15-20 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_over_20_utilization_rate = models.DecimalField( + _("использование оборудования свыше 20 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_under_5_lease_share_itn_percent = models.DecimalField( + _("доля времени работы для ПВН до 5 лет, %"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_5_10_lease_share_itn_percent = models.DecimalField( + _("доля времени работы для ПВН 5-10 лет, %"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_10_15_lease_share_itn_percent = models.DecimalField( + _("доля времени работы для ПВН 10-15 лет, %"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_15_20_lease_share_itn_percent = models.DecimalField( + _("доля времени работы для ПВН 15-20 лет, %"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_over_20_lease_share_itn_percent = models.DecimalField( + _("доля времени работы для ПВН свыше 20 лет, %"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_under_5_weighted_load_factor = models.DecimalField( + _("средневзвешенный коэффициент загрузки до 5 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_5_10_weighted_load_factor = models.DecimalField( + _("средневзвешенный коэффициент загрузки 5-10 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_10_15_weighted_load_factor = models.DecimalField( + _("средневзвешенный коэффициент загрузки 10-15 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_15_20_weighted_load_factor = models.DecimalField( + _("средневзвешенный коэффициент загрузки 15-20 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) + age_over_20_weighted_load_factor = models.DecimalField( + _("средневзвешенный коэффициент загрузки свыше 20 лет"), + max_digits=5, + decimal_places=2, + null=True, + blank=True, + ) # === С ЧПУ по возрасту === cnc_total = models.PositiveIntegerField( @@ -162,6 +303,7 @@ class FormF6Record(ReportingPeriodMixin, TimestampMixin, models.Model): null=True, blank=True, ) + notes = models.TextField(_("примечание"), blank=True, default="") class Meta: verbose_name = _("запись Ф-6") diff --git a/src/apps/form_6/services.py b/src/apps/form_6/services.py index 27d9e91..7088c3a 100644 --- a/src/apps/form_6/services.py +++ b/src/apps/form_6/services.py @@ -7,11 +7,14 @@ """ import logging +from decimal import Decimal, InvalidOperation from typing import Any from apps.core.excel import ( BaseExcelParser, ColumnMapping, + ExcelValidationError, + FieldError, ParseResult, RowData, ) @@ -75,55 +78,209 @@ class FormF6Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF6Record]): OKPO_COLUMN = 1 OGRN_COLUMN = 2 INN_COLUMN = 3 + DATA_START_ROW = 5 def get_column_mappings(self) -> list[ColumnMapping]: return [ # Категоризация ColumnMapping(4, "Код строки", "row_code", field_type="str"), - ColumnMapping(5, "Категория оборудования", "category", field_type="str"), - # Общие данные - ColumnMapping(6, "Всего оборудования", "total_equipment", field_type="int"), + ColumnMapping(5, "Категория", "category", field_type="str"), ColumnMapping( - 7, "Отечественное оборудование", "domestic_equipment", field_type="int" - ), - ColumnMapping( - 8, "Импортное оборудование", "imported_equipment", field_type="int" - ), - # Возрастная структура - ColumnMapping(9, "До 5 лет", "age_under_5", field_type="int"), - ColumnMapping(10, "5-10 лет", "age_5_10", field_type="int"), - ColumnMapping(11, "10-15 лет", "age_10_15", field_type="int"), - ColumnMapping(12, "15-20 лет", "age_15_20", field_type="int"), - ColumnMapping(13, "Свыше 20 лет", "age_over_20", field_type="int"), - # С ЧПУ - ColumnMapping(14, "С ЧПУ всего", "cnc_total", field_type="int"), - ColumnMapping(15, "С ЧПУ до 5 лет", "cnc_under_5", field_type="int"), - ColumnMapping(16, "С ЧПУ 5-10 лет", "cnc_5_10", field_type="int"), - ColumnMapping(17, "С ЧПУ 10-15 лет", "cnc_10_15", field_type="int"), - ColumnMapping(18, "С ЧПУ 15-20 лет", "cnc_15_20", field_type="int"), - ColumnMapping(19, "С ЧПУ свыше 20 лет", "cnc_over_20", field_type="int"), - # Показатели - ColumnMapping( - 20, "Средняя сменность работы", "avg_shift_work", field_type="decimal" - ), - ColumnMapping( - 21, "Коэффициент загрузки", "utilization_rate", field_type="decimal" - ), - ColumnMapping( - 22, "Физический износ, %", "physical_wear_percent", field_type="decimal" - ), - # Потребности - ColumnMapping( - 23, - "Рабочие места без оборудования", - "workplaces_without_equipment", + 6, + "Количество оборудования на конец года ед. Всего", + "total_equipment", field_type="int", ), ColumnMapping( - 24, "Оборудование к замене", "equipment_to_replace", field_type="int" + 7, + "Введенное в эксплуатацию", + "commissioned_equipment", + field_type="int", + ), + ColumnMapping( + 8, + "Выведенное", + "decommissioned_equipment", + field_type="int", + ), + ColumnMapping( + 9, + "Импортное оборудование", + "imported_equipment", + field_type="int", + ), + *self._age_bucket_mappings(10, "До 5 лет", "age_under_5"), + *self._age_bucket_mappings(15, "От 5 до 10 лет", "age_5_10"), + *self._age_bucket_mappings(20, "От 10 до 15 лет", "age_10_15"), + *self._age_bucket_mappings(25, "От 15 до 20 лет", "age_15_20"), + *self._age_bucket_mappings(30, "Свыше 20 лет", "age_over_20"), + ColumnMapping( + 35, "Средняя сменность работы", "avg_shift_work", field_type="decimal" + ), + ColumnMapping( + 36, + "Количество рабочих мест без оборудования", + "workplaces_without_equipment", + field_type="int", + ), + ColumnMapping(37, "Примечание", "notes", field_type="str"), + ] + + @staticmethod + def _age_bucket_mappings( + start_column: int, + age_header: str, + field_prefix: str, + ) -> list[ColumnMapping]: + return [ + ColumnMapping( + start_column, + f"{age_header} Всего", + field_prefix, + field_type="int", + ), + ColumnMapping( + start_column + 1, + "Импортного оборудования", + f"{field_prefix}_imported", + field_type="int", + ), + ColumnMapping( + start_column + 2, + "Коэффициент использования оборудования", + f"{field_prefix}_utilization_rate", + field_type="decimal", + ), + ColumnMapping( + start_column + 3, + "Доля фактического времени работы оборудования", + f"{field_prefix}_lease_share_itn_percent", + field_type="decimal", + ), + ColumnMapping( + start_column + 4, + "Средневзвешенный коэффициент загрузки", + f"{field_prefix}_weighted_load_factor", + field_type="decimal", ), ] + def validate_workbook(self) -> None: + self._validate_mapped_headers(header_rows=(1, 2, 3, 4)) + errors: list[FieldError] = [] + for row_num in range(self.DATA_START_ROW, self._sheet.max_row + 1): + if not self._get_cell_value(row_num, self.ORG_NAME_COLUMN): + continue + self._validate_integer_counts(row_num, errors) + self._validate_equipment_totals(row_num, errors) + + if errors: + raise ExcelValidationError( + "Ф-6 содержит несовместимую схему или противоречивые количества", + errors=errors, + ) + + def _validate_integer_counts( + self, + row_num: int, + errors: list[FieldError], + ) -> None: + int_mappings = [ + mapping + for mapping in self._column_mappings or [] + if mapping.field_type == "int" + ] + for mapping in int_mappings: + raw_value = self._get_cell_value(row_num, mapping.excel_column) + if raw_value in (None, ""): + continue + try: + numeric = Decimal(str(raw_value).replace(",", ".").replace(" ", "")) + except (InvalidOperation, ValueError): + errors.append( + FieldError( + field=mapping.model_field, + message=f"Строка {row_num}: ожидается целое число", + value=raw_value, + ) + ) + continue + if not numeric.is_finite() or numeric != numeric.to_integral_value(): + errors.append( + FieldError( + field=mapping.model_field, + message=f"Строка {row_num}: количество должно быть целым", + value=raw_value, + ) + ) + + def _validate_equipment_totals( + self, + row_num: int, + errors: list[FieldError], + ) -> None: + age_prefixes = ( + "age_under_5", + "age_5_10", + "age_10_15", + "age_15_20", + "age_over_20", + ) + values = { + mapping.model_field: self._convert_value( + self._get_cell_value(row_num, mapping.excel_column), + mapping.field_type, + ) + for mapping in self._column_mappings or [] + } + total = values.get("total_equipment") + age_total = sum(values.get(prefix) or 0 for prefix in age_prefixes) + imported = values.get("imported_equipment") + age_imported = sum( + values.get(f"{prefix}_imported") or 0 for prefix in age_prefixes + ) + + if total is not None and age_total != total: + errors.append( + FieldError( + field="total_equipment", + message=( + f"Строка {row_num}: сумма возрастных групп " + "не равна общему количеству" + ), + value={"total": total, "age_total": age_total}, + ) + ) + if imported is not None and age_imported != imported: + errors.append( + FieldError( + field="imported_equipment", + message=( + f"Строка {row_num}: сумма импортного оборудования " + "по возрастам не равна итогу" + ), + value={"total": imported, "age_total": age_imported}, + ) + ) + for prefix in age_prefixes: + bucket_total = values.get(prefix) + bucket_imported = values.get(f"{prefix}_imported") + if ( + bucket_total is not None + and bucket_imported is not None + and bucket_imported > bucket_total + ): + errors.append( + FieldError( + field=f"{prefix}_imported", + message=( + f"Строка {row_num}: импортное оборудование " + "превышает количество в возрастной группе" + ), + value=bucket_imported, + ) + ) + def get_next_batch_id(self) -> int: return FormF6Service.get_next_batch_id() diff --git a/src/apps/organization/analytics_services.py b/src/apps/organization/analytics_services.py index 1a88986..af4293c 100644 --- a/src/apps/organization/analytics_services.py +++ b/src/apps/organization/analytics_services.py @@ -49,10 +49,6 @@ def _amount(value) -> int: return int(_dec(value)) -def _amount_thousands(value) -> int: - return int(_dec(value) / THOUSAND) - - def _ratio(value) -> float: return round(float(_dec(value)), 1) @@ -64,6 +60,12 @@ def _share_percent(numerator, denominator) -> float: return round(float((_dec(numerator) / denominator_value) * Decimal("100")), 1) +def _nullable_share_percent(numerator, denominator) -> float | None: + if numerator is None or denominator is None or _dec(denominator) == ZERO: + return None + return _share_percent(numerator, denominator) + + def _delta_percent(current, previous) -> float: current_value = _dec(current) previous_value = _dec(previous) @@ -87,8 +89,18 @@ def _direction(current, previous) -> str: return "unchanged" -def _period_rank(report_quarter: int | None) -> int: - return 5 if report_quarter is None else report_quarter +def _period_rank(record) -> int: + """Rank the reporting period represented by a record within its year.""" + report_month = getattr(record, "report_month", None) + if report_month is not None: + return int(report_month) + report_half_year = getattr(record, "report_half_year", None) + if report_half_year is not None: + return int(report_half_year) * 6 + report_quarter = getattr(record, "report_quarter", None) + if report_quarter is not None: + return int(report_quarter) * 3 + return 13 def _pick_record( @@ -106,7 +118,7 @@ def _pick_record( return max( year_records, - key=lambda record: (_period_rank(record.report_quarter), record.created_at), + key=lambda record: (_period_rank(record), record.created_at), ) @@ -331,7 +343,7 @@ class OrganizationAnalyticsService: "assets": "rub_thousands", "capex": "rub_thousands", "rd_expenses": "rub_thousands", - "revenue_per_employee": "rub_per_employee", + "revenue_per_employee": "rub_thousands_per_employee", } @staticmethod @@ -368,23 +380,24 @@ class OrganizationAnalyticsService: if metric == "loans": if f4 is not None and f4.loans_rsbu is not None: return _dec(f4.loans_rsbu) - return _dec(getattr(f2, "borrowings_non_current", ZERO)) + _dec( - getattr(f2, "borrowings_current", ZERO) - ) + return ( + _dec(getattr(f2, "borrowings_non_current", ZERO)) + + _dec(getattr(f2, "borrowings_current", ZERO)) + ) / THOUSAND primary, fallback = field_map.get(metric, (ZERO, ZERO)) - return _dec(primary or fallback) + if primary is not None: + return _dec(primary) + return _dec(fallback) / THOUSAND @classmethod def _economics_display_value(cls, metric: str, f2, f4, f1) -> int: if metric == "revenue_per_employee": - employees = _dec(getattr(f1, "avg_payroll_employees", ZERO)) - if employees == ZERO: - employees = _dec(getattr(f1, "avg_employees", ZERO)) + employees = _dec(getattr(f1, "avg_employees", ZERO)) if employees == ZERO: return 0 return _amount(cls._economics_metric_value("revenue", f2, f4) / employees) - return _amount_thousands(cls._economics_metric_value(metric, f2, f4)) + return _amount(cls._economics_metric_value(metric, f2, f4)) @classmethod def get_economics( @@ -405,18 +418,20 @@ class OrganizationAnalyticsService: cls._f1_records(organization), from_year, to_year ) - periods = sorted(set(f2_by_year) | set(f4_by_year) | set(f1_by_year)) + periods = sorted(set(f2_by_year) | set(f4_by_year)) if not periods: return { "organization_id": str(organization.id), "group": group, "periods": [], + "report_periods": [], "data_available": False, "message": "Данные отсутствуют", "kpis": {}, "series": [], "ratios": [], "ratio_normatives": ECONOMICS_RATIO_NORMATIVES, + "ratios_report_period": None, } metric_units = cls._economics_metric_units() @@ -434,10 +449,46 @@ class OrganizationAnalyticsService: f1_by_year.get(period), ) + def profitability_ratio(report_year: int, denominator_field: str): + f4_record = f4_by_year.get(report_year) + f2_record = f2_by_year.get(report_year) + return _nullable_share_percent( + ( + f4_record.net_profit_rsbu + if f4_record is not None and f4_record.net_profit_rsbu is not None + else None + ), + ( + getattr(f2_record, denominator_field, None) + if f2_record is not None + else None + ), + ) + + latest_f2 = f2_by_year.get(last_period) + return { "organization_id": str(organization.id), "group": group, "periods": periods, + "report_periods": [ + { + "year": year, + "report_half_year": getattr( + f4_by_year.get(year), "report_half_year", None + ), + } + for year in periods + ], + "ratios_report_period": ( + { + "financial_form": "F-2", + "report_year": latest_f2.report_year, + "report_quarter": latest_f2.report_quarter, + } + if latest_f2 is not None + else None + ), "data_available": True, "message": None, "kpis": { @@ -475,9 +526,20 @@ class OrganizationAnalyticsService: "ratios": [ { "period": report_year, - "ros": _ratio(getattr(f4_by_year.get(report_year), "ros", ZERO)), - "roa": _ratio(getattr(f4_by_year.get(report_year), "roa", ZERO)), - "roe": _ratio(getattr(f4_by_year.get(report_year), "roe", ZERO)), + "ros": _share_percent( + cls._economics_metric_value( + "net_profit", + f2_by_year.get(report_year), + f4_by_year.get(report_year), + ), + cls._economics_metric_value( + "revenue", + f2_by_year.get(report_year), + f4_by_year.get(report_year), + ), + ), + "roa": profitability_ratio(report_year, "total_assets"), + "roe": profitability_ratio(report_year, "total_equity"), "ebitda_margin": _share_percent( cls._economics_metric_value( "ebitda", @@ -504,95 +566,103 @@ class OrganizationAnalyticsService: report_year: int, history_years: int, ) -> dict[str, object]: + f1_records = cls._f1_records(organization) f3_records = cls._f3_records(organization) - report_year = _resolve_report_year(f3_records, report_year) - current_f3 = cls._require_record( - _pick_record(f3_records, report_year), + report_year = _resolve_report_year(f1_records, report_year) + current_f1 = cls._require_record( + _pick_record(f1_records, report_year), entity="Personnel", ) - current_f1 = _pick_record(cls._f1_records(organization), report_year) + current_f3 = _pick_record(f3_records, report_year) years = list(range(report_year - history_years + 1, report_year + 1)) - history_records = _best_records_by_year(f3_records, years[0], years[-1]) - history_f1_records = _best_records_by_year( - cls._f1_records(organization), years[0], years[-1] + history_f1_records = _best_records_by_year(f1_records, years[0], years[-1]) + history_f3_records = _best_records_by_year(f3_records, years[0], years[-1]) + + def optional_int(record, field_name: str) -> int | None: + value = getattr(record, field_name, None) if record is not None else None + return int(_dec(value)) if value is not None else None + + def age_distribution_for(record) -> list[dict[str, int]]: + return [ + { + "age_group": "under_30", + "employees_count": (optional_int(record, "employees_under_20") or 0) + + (optional_int(record, "employees_20_29") or 0), + }, + { + "age_group": "30_50", + "employees_count": (optional_int(record, "employees_30_39") or 0) + + (optional_int(record, "employees_40_49") or 0), + }, + { + "age_group": "over_50", + "employees_count": (optional_int(record, "employees_50_59") or 0) + + (optional_int(record, "employees_over_60") or 0), + }, + ] + + age_distribution = age_distribution_for(current_f3) + average_age = ( + _ratio(current_f3.average_employee_age) + if current_f3 is not None and current_f3.average_employee_age is not None + else None ) - average_employees = int(_dec(current_f3.avg_employees)) - under_30 = int(average_employees * 0.29) - from_30_to_50 = int(average_employees * 0.49) - over_50 = max(0, average_employees - under_30 - from_30_to_50) - - age_distribution = [ - {"age_group": "under_30", "employees_count": under_30}, - {"age_group": "30_50", "employees_count": from_30_to_50}, - {"age_group": "over_50", "employees_count": over_50}, - ] - return { "organization_id": str(organization.id), "report_year": report_year, - "average_age": _weighted_average_age(age_distribution), + "report_month": current_f1.report_month, + "average_age": average_age, "headcount": { - "average_employees": average_employees, + "average_employees": int(_dec(current_f1.avg_employees)), "avg_payroll_employees": int( _dec(getattr(current_f1, "avg_payroll_employees", ZERO)) ), - "payroll_fund": _amount(getattr(current_f1, "payroll_fund", ZERO)), - "production_workers": int(_dec(current_f3.production_workers)), - "engineering_workers": int(_dec(current_f3.engineering_workers)), - "administrative_workers": int(_dec(current_f3.administrative_workers)), - "workers_needed": int(_dec(current_f3.workers_needed)), + "payroll_fund": _amount( + _dec(getattr(current_f1, "payroll_fund", ZERO)) * THOUSAND + ), + "production_workers": optional_int(current_f3, "production_workers"), + "engineering_workers": optional_int(current_f3, "engineering_workers"), + "administrative_workers": optional_int( + current_f3, "administrative_workers" + ), + "workers_needed": optional_int(current_f3, "workers_needed"), }, "age_distribution": age_distribution, "history": [ { "year": year, - "average_employees": int(_dec(history_records[year].avg_employees)), + "report_month": history_f1_records[year].report_month, + "average_employees": int( + _dec(history_f1_records[year].avg_employees) + ), "avg_payroll_employees": int( _dec( getattr( - history_f1_records.get(year), + history_f1_records[year], "avg_payroll_employees", ZERO, ) ) ), - "average_age": _weighted_average_age( - [ - { - "age_group": "under_30", - "employees_count": int( - _dec(history_records[year].avg_employees) - * Decimal("0.29") - ), - }, - { - "age_group": "30_50", - "employees_count": int( - _dec(history_records[year].avg_employees) - * Decimal("0.49") - ), - }, - { - "age_group": "over_50", - "employees_count": int( - _dec(history_records[year].avg_employees) - * Decimal("0.22") - ), - }, - ] + "average_age": ( + _ratio(history_f3_records[year].average_employee_age) + if year in history_f3_records + and history_f3_records[year].average_employee_age is not None + else None ), "payroll_fund": _amount( - getattr(history_f1_records.get(year), "payroll_fund", ZERO) + _dec(getattr(history_f1_records[year], "payroll_fund", ZERO)) + * THOUSAND ), } for year in years - if year in history_records + if year in history_f1_records ], } @classmethod - def get_equipment( + def get_equipment( # noqa: C901 cls, *, organization: Organization, @@ -616,6 +686,40 @@ class OrganizationAnalyticsService: report_year=report_year, ) + def weighted_metric(records, value_field: str, weight_field: str): + weighted_sum = ZERO + total_weight = 0 + for record in records: + value = getattr(record, value_field, None) + weight = int(_dec(getattr(record, weight_field, ZERO))) + if value is None or weight <= 0: + continue + weighted_sum += _dec(value) * weight + total_weight += weight + if total_weight == 0: + return None + return weighted_sum / Decimal(str(total_weight)) + + def record_age_metric(record, suffix: str): + weighted_sum = ZERO + total_weight = 0 + for prefix in ( + "age_under_5", + "age_5_10", + "age_10_15", + "age_15_20", + "age_over_20", + ): + value = getattr(record, f"{prefix}_{suffix}", None) + weight = int(_dec(getattr(record, prefix, ZERO))) + if value is None or weight <= 0: + continue + weighted_sum += _dec(value) * weight + total_weight += weight + if total_weight == 0: + return None + return weighted_sum / Decimal(str(total_weight)) + if f6_records: age_distribution = [ { @@ -690,51 +794,55 @@ class OrganizationAnalyticsService: "15_20_years": "age_15_20", "over_20_years": "age_over_20", } - total_age_units = sum(item["units_count"] for item in age_distribution) for item in age_distribution: bucket = item["bucket"] if f6_records: - bucket_units_by_record = [ - int(_dec(getattr(record, bucket_fields[bucket], ZERO))) - for record in f6_records - ] + bucket_field = bucket_fields[bucket] imported_equipment = sum( - round( - int(_dec(record.imported_equipment)) - * units - / max(1, int(_dec(record.total_equipment))) - ) - for record, units in zip( - f6_records, bucket_units_by_record, strict=True - ) + int(_dec(getattr(record, f"{bucket_field}_imported", ZERO))) + for record in f6_records + ) + utilization_rate = weighted_metric( + f6_records, + f"{bucket_field}_utilization_rate", + bucket_field, + ) + lease_share = weighted_metric( + f6_records, + f"{bucket_field}_lease_share_itn_percent", + bucket_field, + ) + weighted_load_factor = weighted_metric( + f6_records, + f"{bucket_field}_weighted_load_factor", + bucket_field, ) - weighted_units = sum(bucket_units_by_record) - utilization_rate = sum( - _dec(record.utilization_rate) * units - for record, units in zip( - f6_records, bucket_units_by_record, strict=True - ) - ) / max(1, weighted_units) - weighted_wear = sum( - _dec(record.physical_wear_percent) * units - for record, units in zip( - f6_records, bucket_units_by_record, strict=True - ) - ) / max(1, weighted_units) else: - imported_equipment = round( - int(_dec(current_f3.imported_equipment)) - * item["units_count"] - / max(1, total_age_units) + imported_equipment = None + utilization_rate = ( + _dec(current_f3.utilization_rate) + if current_f3.utilization_rate is not None + else None ) - utilization_rate = _dec(current_f3.utilization_rate) - weighted_wear = _dec(current_f3.physical_wear_percent) + lease_share = None + weighted_load_factor = None item.update( { "imported_equipment": imported_equipment, - "utilization_rate": round(float(utilization_rate / 100), 2), - "lease_share_itn_percent": None, - "weighted_wear_percent": _ratio(weighted_wear), + "utilization_rate": ( + round(float(utilization_rate / 100), 2) + if utilization_rate is not None + else None + ), + "lease_share_itn_percent": ( + _ratio(lease_share) if lease_share is not None else None + ), + "weighted_wear_percent": None, + "weighted_load_factor": ( + round(float(weighted_load_factor), 2) + if weighted_load_factor is not None + else None + ), } ) @@ -743,11 +851,13 @@ class OrganizationAnalyticsService: { "equipment_category": record.category, "total_equipment": record.total_equipment, - "domestic_equipment": record.domestic_equipment, + "domestic_equipment": None, "imported_equipment": record.imported_equipment, - "physical_wear_percent": record.physical_wear_percent, - "utilization_rate": record.utilization_rate, - "lease_share_itn_percent": None, + "physical_wear_percent": None, + "utilization_rate": record_age_metric(record, "utilization_rate"), + "lease_share_itn_percent": record_age_metric( + record, "lease_share_itn_percent" + ), } for record in f6_records ] @@ -776,47 +886,115 @@ class OrganizationAnalyticsService: .order_by("equipment_category") ) - f3_by_year = _best_records_by_year( - f3_records, - min(record.report_year for record in f3_records), - max(record.report_year for record in f3_records), - ) - dynamics_years = sorted(f3_by_year) - commissioned_by_year = { - year: FormF5Record.objects.filter( + all_f6_records = list( + FormF6Record.objects.filter( organization=organization, is_active_version=True, - report_year=year, - commissioning_date__year=year, - ).count() - for year in dynamics_years - } - decommissioned_by_year = { - year: FormF5Record.objects.filter( - organization=organization, - is_active_version=True, - report_year=year, ) - .filter(Q(is_operational=False) | Q(requires_replacement=True)) - .count() - for year in dynamics_years - } + ) + f6_by_year: dict[int, list[FormF6Record]] = defaultdict(list) + for record in all_f6_records: + f6_by_year[record.report_year].append(record) + + if f6_by_year: + dynamics_years = sorted(f6_by_year) + total_by_year = { + year: sum(int(_dec(record.total_equipment)) for record in year_records) + for year, year_records in f6_by_year.items() + } + commissioned_by_year = { + year: sum( + int(_dec(record.commissioned_equipment)) for record in year_records + ) + for year, year_records in f6_by_year.items() + } + decommissioned_by_year = { + year: sum( + int(_dec(record.decommissioned_equipment)) + for record in year_records + ) + for year, year_records in f6_by_year.items() + } + else: + f3_by_year = _best_records_by_year( + f3_records, + min(record.report_year for record in f3_records), + max(record.report_year for record in f3_records), + ) + dynamics_years = sorted(f3_by_year) + total_by_year = { + year: int(_dec(f3_by_year[year].total_equipment)) + for year in dynamics_years + } + commissioned_by_year = { + year: FormF5Record.objects.filter( + organization=organization, + is_active_version=True, + report_year=year, + commissioning_date__year=year, + ).count() + for year in dynamics_years + } + decommissioned_by_year = { + year: FormF5Record.objects.filter( + organization=organization, + is_active_version=True, + report_year=year, + ) + .filter(Q(is_operational=False) | Q(requires_replacement=True)) + .count() + for year in dynamics_years + } + + total_equipment = ( + sum(int(_dec(record.total_equipment)) for record in f6_records) + if f6_records + else int(_dec(current_f3.total_equipment)) + ) + imported_equipment = ( + sum(int(_dec(record.imported_equipment)) for record in f6_records) + if f6_records + else int(_dec(current_f3.imported_equipment)) + ) + average_shift = ( + weighted_metric(f6_records, "avg_shift_work", "total_equipment") + if f6_records + else getattr(current_f3, "avg_shift_work", None) + ) return { "organization_id": str(organization.id), "report_year": report_year, "summary": { - "total_equipment": int(_dec(current_f3.total_equipment)), - "machine_tools_and_equipment": int(_dec(current_f3.total_equipment)), - "domestic_equipment": int(_dec(current_f3.domestic_equipment)), - "imported_equipment": int(_dec(current_f3.imported_equipment)), - "physical_wear_percent": _ratio(current_f3.physical_wear_percent), - "weighted_wear_percent": _ratio(current_f3.physical_wear_percent), - "utilization_rate": round( - float(_dec(current_f3.utilization_rate) / Decimal("100")), 2 + "total_equipment": total_equipment, + "machine_tools_and_equipment": int( + _dec( + getattr(current_f3, "machine_tools_and_equipment", None) + or current_f3.total_equipment + ) + ), + "domestic_equipment": ( + None if f6_records else int(_dec(current_f3.domestic_equipment)) + ), + "imported_equipment": imported_equipment, + "physical_wear_percent": _ratio(current_f3.physical_wear_percent), + "weighted_wear_percent": None, + "utilization_rate": ( + round( + float(_dec(current_f3.utilization_rate) / Decimal("100")), + 2, + ) + if current_f3.utilization_rate is not None + else None + ), + "avg_shift_work": ( + _ratio(average_shift) if average_shift is not None else None + ), + "equipment_needed": ( + int(_dec(current_f3.equipment_needed)) + if current_f3.equipment_needed is not None + else None ), - "avg_shift_work": _ratio(current_f3.avg_shift_work), - "equipment_needed": int(_dec(current_f3.equipment_needed)), "average_age_years": _weighted_average_age(age_distribution), "commissioned_equipment": commissioned_by_year.get(report_year, 0), "decommissioned_equipment": decommissioned_by_year.get(report_year, 0), @@ -826,16 +1004,28 @@ class OrganizationAnalyticsService: { "category": row["equipment_category"] or "Без категории", "total_equipment": int(_dec(row["total_equipment"])), - "domestic_equipment": int(_dec(row["domestic_equipment"])), + "domestic_equipment": ( + int(_dec(row["domestic_equipment"])) + if row["domestic_equipment"] is not None + else None + ), "imported_equipment": int(_dec(row["imported_equipment"])), - "physical_wear_percent": _ratio(row["physical_wear_percent"]), - "weighted_wear_percent": _ratio(row["physical_wear_percent"]), + "physical_wear_percent": ( + _ratio(row["physical_wear_percent"]) + if row["physical_wear_percent"] is not None + else None + ), + "weighted_wear_percent": None, "utilization_rate": round( float(_dec(row.get("utilization_rate")) / Decimal("100")), 2 ) if row.get("utilization_rate") is not None else None, - "lease_share_itn_percent": row.get("lease_share_itn_percent"), + "lease_share_itn_percent": ( + _ratio(row["lease_share_itn_percent"]) + if row.get("lease_share_itn_percent") is not None + else None + ), } for row in category_rows ], @@ -846,7 +1036,7 @@ class OrganizationAnalyticsService: "points": [ { "period": year, - "value": int(_dec(f3_by_year[year].total_equipment)), + "value": total_by_year[year], } for year in dynamics_years ], @@ -921,72 +1111,132 @@ class OrganizationAnalyticsService: metrics[key] += _dec(row_metrics[key]) return {"period": period, "metrics": metrics} + @classmethod + def _build_product_base_rows( + cls, records: list[FormF1Record], *, suffix: str + ) -> list[dict[str, object]]: + base_rows = [] + for record in records: + month = record.report_month + explicit_quarter = record.report_quarter + quarter = explicit_quarter + if quarter is None and month is not None: + quarter = ((month - 1) // 3) + 1 + base_rows.append( + { + "month": month, + "quarter": quarter, + "explicit_quarter": explicit_quarter, + "period": ( + f"{record.report_year}-{month:02d}" + if month is not None + else ( + f"{record.report_year}-Q{quarter}" + if quarter is not None + else str(record.report_year) + ) + ), + "metrics": cls._f1_metric_bundle(record, suffix=suffix), + } + ) + return base_rows + + @classmethod + def _build_quarterly_product_rows( + cls, base_rows: list[dict[str, object]], report_year: int + ) -> list[dict[str, object]]: + grouped_rows = [] + for quarter in range(1, 5): + quarter_rows = [row for row in base_rows if row["quarter"] == quarter] + if quarter_rows: + grouped_rows.append( + cls._aggregate_product_metrics( + quarter_rows, + f"{report_year}-Q{quarter}", + ) + ) + return grouped_rows or [ + cls._aggregate_product_metrics(base_rows, str(report_year)) + ] + + @classmethod + def _build_semiannual_product_rows( + cls, base_rows: list[dict[str, object]], report_year: int + ) -> list[dict[str, object]]: + grouped_rows = [] + periods = ( + (1, 2, f"{report_year}-H1"), + (3, 4, f"{report_year}-H2"), + ) + for quarter_start, quarter_end, period in periods: + half_rows = [ + row + for row in base_rows + if row["quarter"] is not None + and quarter_start <= row["quarter"] <= quarter_end + ] + if half_rows: + grouped_rows.append(cls._aggregate_product_metrics(half_rows, period)) + return grouped_rows or [ + cls._aggregate_product_metrics(base_rows, str(report_year)) + ] + + @classmethod + def _build_monthly_product_rows( + cls, base_rows: list[dict[str, object]], report_year: int + ) -> list[dict[str, object]]: + real_monthly_rows = [ + row + for row in base_rows + if row["month"] is not None and row["explicit_quarter"] is None + ] + if real_monthly_rows: + grouped_rows = [] + for month in range(1, 13): + month_rows = [row for row in real_monthly_rows if row["month"] == month] + if month_rows: + grouped_rows.append( + cls._aggregate_product_metrics( + month_rows, + f"{report_year}-{month:02d}", + ) + ) + return grouped_rows + + monthly_rows = [] + month_map = {1: (1, 2, 3), 2: (4, 5, 6), 3: (7, 8, 9), 4: (10, 11, 12)} + for row in base_rows: + quarter = row["quarter"] + if quarter is None: + monthly_rows.append(row) + continue + for month in month_map.get(quarter, ()): + month_metrics = { + key: value / Decimal("3") for key, value in row["metrics"].items() + } + monthly_rows.append( + { + "period": f"{report_year}-{month:02d}", + "metrics": month_metrics, + } + ) + return sorted(monthly_rows, key=lambda row: row["period"]) + @classmethod def _build_product_frequency_rows( cls, records: list[FormF1Record], *, suffix: str, frequency: str ) -> list[dict[str, object]]: - base_rows = [ - { - "quarter": record.report_quarter, - "period": ( - str(record.report_year) - if record.report_quarter is None - else f"{record.report_year}-Q{record.report_quarter}" - ), - "metrics": cls._f1_metric_bundle(record, suffix=suffix), - } - for record in records - ] - - if frequency == "quarterly": - return base_rows - - if frequency == "annual": - return [ - cls._aggregate_product_metrics(base_rows, str(records[0].report_year)) - ] - - if frequency == "semiannual": - grouped_rows: list[dict[str, object]] = [] - periods = ( - (1, 2, f"{records[0].report_year}-H1"), - (3, 4, f"{records[0].report_year}-H2"), - ) - for quarter_start, quarter_end, period in periods: - half_rows = [ - row - for row in base_rows - if row["quarter"] is not None - and quarter_start <= row["quarter"] <= quarter_end - ] - if half_rows: - grouped_rows.append( - cls._aggregate_product_metrics(half_rows, period) - ) - return grouped_rows or [ - cls._aggregate_product_metrics(base_rows, str(records[0].report_year)) - ] + report_year = records[0].report_year + base_rows = cls._build_product_base_rows(records, suffix=suffix) if frequency == "monthly": - monthly_rows: list[dict[str, object]] = [] - month_map = {1: (1, 2, 3), 2: (4, 5, 6), 3: (7, 8, 9), 4: (10, 11, 12)} - for row in base_rows: - quarter = row["quarter"] - if quarter is None: - monthly_rows.append(row) - continue - for month in month_map.get(quarter, ()): - month_metrics = { - key: value / Decimal("3") - for key, value in row["metrics"].items() - } - monthly_rows.append( - { - "period": f"{records[0].report_year}-{month:02d}", - "metrics": month_metrics, - } - ) - return monthly_rows + return cls._build_monthly_product_rows(base_rows, report_year) + if frequency == "quarterly": + return cls._build_quarterly_product_rows(base_rows, report_year) + if frequency == "semiannual": + return cls._build_semiannual_product_rows(base_rows, report_year) + if frequency == "annual": + return [cls._aggregate_product_metrics(base_rows, str(report_year))] return base_rows @@ -1015,9 +1265,7 @@ class OrganizationAnalyticsService: if not records: raise NotFoundError(message="Products data is not available") - records.sort( - key=lambda record: (_period_rank(record.report_quarter), record.created_at) - ) + records.sort(key=lambda record: (_period_rank(record), record.created_at)) suffix = "actual" if price_mode == "actual" else "fixed" frequency_rows = cls._build_product_frequency_rows( records, suffix=suffix, frequency=frequency @@ -1030,6 +1278,7 @@ class OrganizationAnalyticsService: "report_year": report_year, "frequency": frequency, "price_mode": price_mode, + "summary_period": current["period"], "summary": { "military_output_amount": _amount( current_metrics["military_output_amount"] diff --git a/src/apps/organization/contract_serializers.py b/src/apps/organization/contract_serializers.py index 24311e1..1e9f0aa 100644 --- a/src/apps/organization/contract_serializers.py +++ b/src/apps/organization/contract_serializers.py @@ -78,8 +78,8 @@ class EconomicsMetricSeriesSerializer(serializers.Serializer): class EconomicsRatioSerializer(serializers.Serializer): period = serializers.IntegerField() ros = serializers.FloatField() - roa = serializers.FloatField() - roe = serializers.FloatField() + roa = serializers.FloatField(allow_null=True) + roe = serializers.FloatField(allow_null=True) ebitda_margin = serializers.FloatField() @@ -90,10 +90,23 @@ class EconomicsRatioNormativesSerializer(serializers.Serializer): ebitda_margin = serializers.FloatField(allow_null=True) +class EconomicsReportPeriodSerializer(serializers.Serializer): + year = serializers.IntegerField() + report_half_year = serializers.IntegerField(allow_null=True) + + +class EconomicsRatiosReportPeriodSerializer(serializers.Serializer): + financial_form = serializers.CharField() + report_year = serializers.IntegerField() + report_quarter = serializers.IntegerField(allow_null=True) + + class EconomicsResponseSerializer(serializers.Serializer): organization_id = serializers.UUIDField() group = serializers.CharField() periods = serializers.ListField(child=serializers.IntegerField()) + report_periods = EconomicsReportPeriodSerializer(many=True) + ratios_report_period = EconomicsRatiosReportPeriodSerializer(allow_null=True) data_available = serializers.BooleanField() message = serializers.CharField(allow_null=True) kpis = EconomicsKpisSerializer() @@ -106,10 +119,10 @@ class PersonnelHeadcountSerializer(serializers.Serializer): average_employees = serializers.IntegerField() avg_payroll_employees = serializers.IntegerField() payroll_fund = serializers.IntegerField() - production_workers = serializers.IntegerField() - engineering_workers = serializers.IntegerField() - administrative_workers = serializers.IntegerField() - workers_needed = serializers.IntegerField() + production_workers = serializers.IntegerField(allow_null=True) + engineering_workers = serializers.IntegerField(allow_null=True) + administrative_workers = serializers.IntegerField(allow_null=True) + workers_needed = serializers.IntegerField(allow_null=True) class PersonnelAgeDistributionSerializer(serializers.Serializer): @@ -119,16 +132,18 @@ class PersonnelAgeDistributionSerializer(serializers.Serializer): class PersonnelHistorySerializer(serializers.Serializer): year = serializers.IntegerField() + report_month = serializers.IntegerField(allow_null=True) average_employees = serializers.IntegerField() avg_payroll_employees = serializers.IntegerField() - average_age = serializers.FloatField() + average_age = serializers.FloatField(allow_null=True) payroll_fund = serializers.IntegerField() class PersonnelResponseSerializer(serializers.Serializer): organization_id = serializers.UUIDField() report_year = serializers.IntegerField() - average_age = serializers.FloatField() + report_month = serializers.IntegerField(allow_null=True) + average_age = serializers.FloatField(allow_null=True) headcount = PersonnelHeadcountSerializer() age_distribution = PersonnelAgeDistributionSerializer(many=True) history = PersonnelHistorySerializer(many=True) @@ -137,13 +152,13 @@ class PersonnelResponseSerializer(serializers.Serializer): class EquipmentSummarySerializer(serializers.Serializer): total_equipment = serializers.IntegerField() machine_tools_and_equipment = serializers.IntegerField() - domestic_equipment = serializers.IntegerField() + domestic_equipment = serializers.IntegerField(allow_null=True) imported_equipment = serializers.IntegerField() physical_wear_percent = serializers.FloatField() - weighted_wear_percent = serializers.FloatField() - utilization_rate = serializers.FloatField() - avg_shift_work = serializers.FloatField() - equipment_needed = serializers.IntegerField() + weighted_wear_percent = serializers.FloatField(allow_null=True) + utilization_rate = serializers.FloatField(allow_null=True) + avg_shift_work = serializers.FloatField(allow_null=True) + equipment_needed = serializers.IntegerField(allow_null=True) average_age_years = serializers.FloatField() commissioned_equipment = serializers.IntegerField() decommissioned_equipment = serializers.IntegerField() @@ -152,19 +167,20 @@ class EquipmentSummarySerializer(serializers.Serializer): class EquipmentAgeDistributionSerializer(serializers.Serializer): bucket = serializers.CharField() units_count = serializers.IntegerField() - imported_equipment = serializers.IntegerField() - utilization_rate = serializers.FloatField() + imported_equipment = serializers.IntegerField(allow_null=True) + utilization_rate = serializers.FloatField(allow_null=True) lease_share_itn_percent = serializers.FloatField(allow_null=True) - weighted_wear_percent = serializers.FloatField() + weighted_wear_percent = serializers.FloatField(allow_null=True) + weighted_load_factor = serializers.FloatField(allow_null=True) class EquipmentCategorySerializer(serializers.Serializer): category = serializers.CharField() total_equipment = serializers.IntegerField() - domestic_equipment = serializers.IntegerField() + domestic_equipment = serializers.IntegerField(allow_null=True) imported_equipment = serializers.IntegerField() - physical_wear_percent = serializers.FloatField() - weighted_wear_percent = serializers.FloatField() + physical_wear_percent = serializers.FloatField(allow_null=True) + weighted_wear_percent = serializers.FloatField(allow_null=True) utilization_rate = serializers.FloatField(allow_null=True) lease_share_itn_percent = serializers.FloatField(allow_null=True) @@ -236,6 +252,7 @@ class ProductsResponseSerializer(serializers.Serializer): report_year = serializers.IntegerField() frequency = serializers.CharField() price_mode = serializers.CharField() + summary_period = serializers.CharField() summary = ProductsSummarySerializer() production_series = ProductsProductionSeriesSerializer(many=True) sales_series = ProductsSalesSeriesSerializer(many=True) diff --git a/src/apps/user/serializers.py b/src/apps/user/serializers.py index 6c79f9d..57733fa 100644 --- a/src/apps/user/serializers.py +++ b/src/apps/user/serializers.py @@ -7,6 +7,7 @@ from rest_framework import serializers from rest_framework.validators import UniqueValidator from .models import Profile +from .services import UserService User = get_user_model() @@ -126,25 +127,14 @@ class CurrentUserSerializer(serializers.ModelSerializer): @staticmethod def get_role(obj) -> str: - if obj.is_superuser: - return "admin" - if obj.is_staff: - return "staff" - return "user" + return UserService.get_user_role(obj) def get_role_label(self, obj) -> str: - labels = { - "admin": "Администратор системы", - "staff": "Сотрудник системы", - "user": "Пользователь системы", - } - return labels[self.get_role(obj)] + return UserService.get_role_label(self.get_role(obj)) @staticmethod def get_capabilities(obj) -> dict[str, bool]: - return { - "can_access_admin_page": bool(obj.is_staff or obj.is_superuser), - } + return UserService.get_user_capabilities(obj) class Meta: model = User @@ -169,6 +159,11 @@ class UserManagementSerializer(serializers.Serializer): email = serializers.EmailField(read_only=True) phone = serializers.CharField(read_only=True, allow_null=True) is_active = serializers.BooleanField(read_only=True) + is_verified = serializers.BooleanField(read_only=True) + role = serializers.SerializerMethodField() + role_label = serializers.SerializerMethodField() + capabilities = serializers.SerializerMethodField() + profile = CurrentUserProfileSerializer(read_only=True) first_name = serializers.SerializerMethodField() middle_name = serializers.SerializerMethodField() last_name = serializers.SerializerMethodField() @@ -196,6 +191,17 @@ class UserManagementSerializer(serializers.Serializer): def get_last_name(self, obj: User) -> str: return self._get_profile_value(getattr(obj, "profile", None), "last_name") + @staticmethod + def get_role(obj: User) -> str: + return UserService.get_user_role(obj) + + def get_role_label(self, obj: User) -> str: + return UserService.get_role_label(self.get_role(obj)) + + @staticmethod + def get_capabilities(obj: User) -> dict[str, bool]: + return UserService.get_user_capabilities(obj) + @staticmethod def _get_latest_job(obj: User) -> Any | None: return getattr(obj, "latest_job", None) @@ -243,6 +249,157 @@ class UserManagementSerializer(serializers.Serializer): return job.is_successful +class AdminUserListResponseSerializer(serializers.Serializer): + """Пагинированный ответ списка пользователей для административного UI.""" + + count = serializers.IntegerField(read_only=True) + next = serializers.URLField(read_only=True, allow_null=True) + previous = serializers.URLField(read_only=True, allow_null=True) + results = UserManagementSerializer(many=True, read_only=True) + + +class ManagedUserSerializer(serializers.ModelSerializer): + """Короткий ответ мутаций административного управления пользователями.""" + + role = serializers.SerializerMethodField() + role_label = serializers.SerializerMethodField() + + class Meta: + model = User + fields = ( + "id", + "username", + "email", + "phone", + "role", + "role_label", + "is_active", + ) + read_only_fields = fields + + @staticmethod + def get_role(obj: User) -> str: + return UserService.get_user_role(obj) + + def get_role_label(self, obj: User) -> str: + return UserService.get_role_label(self.get_role(obj)) + + +class AdminUserCreateSerializer(serializers.ModelSerializer): + """Входные данные для создания пользователя администратором.""" + + password = serializers.CharField( + write_only=True, + min_length=8, + help_text="Пароль (минимум 8 символов)", + ) + role = serializers.ChoiceField( + choices=UserService.ROLE_CHOICES, + default=UserService.ROLE_USER, + help_text="Прикладная роль пользователя", + ) + first_name = serializers.CharField(max_length=50, allow_blank=False) + middle_name = serializers.CharField( + max_length=50, + required=False, + allow_blank=True, + ) + last_name = serializers.CharField(max_length=50, allow_blank=False) + + class Meta: + model = User + fields = ( + "email", + "username", + "phone", + "password", + "role", + "is_active", + "is_verified", + "first_name", + "middle_name", + "last_name", + ) + extra_kwargs = { + "email": { + "validators": [UniqueValidator(queryset=User.objects.all())], + }, + "username": { + "validators": [UniqueValidator(queryset=User.objects.all())], + }, + } + + +class AdminUserUpdateSerializer(serializers.ModelSerializer): + """Входные данные частичного обновления пользователя администратором.""" + + password = serializers.CharField( + write_only=True, + required=False, + min_length=8, + help_text="Новый пароль (опционально)", + ) + role = serializers.ChoiceField( + choices=UserService.ROLE_CHOICES, + required=False, + help_text="Прикладная роль пользователя", + ) + first_name = serializers.CharField( + max_length=50, + required=False, + allow_blank=True, + ) + middle_name = serializers.CharField( + max_length=50, + required=False, + allow_blank=True, + ) + last_name = serializers.CharField( + max_length=50, + required=False, + allow_blank=True, + ) + + class Meta: + model = User + fields = ( + "email", + "username", + "phone", + "password", + "role", + "is_active", + "is_verified", + "first_name", + "middle_name", + "last_name", + ) + + def validate(self, attrs): + profile_fields = {"first_name", "middle_name", "last_name"} + if not any(field in attrs for field in profile_fields): + return attrs + + profile = getattr(self.instance, "profile", None) if self.instance else None + first_name = attrs.get( + "first_name", + profile.first_name if profile is not None else None, + ) + last_name = attrs.get( + "last_name", + profile.last_name if profile is not None else None, + ) + + errors = {} + if not str(first_name or "").strip(): + errors["first_name"] = "Обязательное поле." + if not str(last_name or "").strip(): + errors["last_name"] = "Обязательное поле." + if errors: + raise serializers.ValidationError(errors) + return attrs + + class UserUpdateSerializer(serializers.ModelSerializer): """Сериализатор для обновления данных пользователя""" diff --git a/src/apps/user/services.py b/src/apps/user/services.py index de36d1b..7474116 100644 --- a/src/apps/user/services.py +++ b/src/apps/user/services.py @@ -2,7 +2,9 @@ from typing import Any from apps.core.exceptions import NotFoundError from django.contrib.auth import get_user_model +from django.contrib.auth.models import Group from django.db import transaction +from django.db.models import F, Q from rest_framework_simplejwt.tokens import RefreshToken from .models import Profile @@ -13,6 +15,14 @@ User = get_user_model() class UserService: """Сервисный слой для работы с пользователями""" + ROLE_USER = "user" + ROLE_ADMIN = "admin" + ROLE_CHOICES = ( + (ROLE_USER, "Пользователь"), + (ROLE_ADMIN, "Администратор"), + ) + ROLE_LABELS = dict(ROLE_CHOICES) + @classmethod def create_user( cls, *, email: str, username: str, password: str, **extra_fields @@ -84,6 +94,74 @@ class UserService: except User.DoesNotExist: return None + @classmethod + def get_users_queryset(cls): + """Базовый queryset для административного управления пользователями.""" + return ( + User.objects.all() + .select_related("profile") + .prefetch_related("groups") + .order_by("-created_at") + ) + + @classmethod + def get_filtered_users_queryset( + cls, + *, + search: str = "", + ordering: str = "", + ): + """Фильтрует и сортирует пользователей для административного API.""" + queryset = cls.get_users_queryset() + search_term = search.strip() + + if search_term: + queryset = queryset.filter( + Q(username__icontains=search_term) + | Q(email__icontains=search_term) + | Q(phone__icontains=search_term) + | Q(profile__first_name__icontains=search_term) + | Q(profile__mid_name__icontains=search_term) + | Q(profile__last_name__icontains=search_term) + ).distinct() + + ordering_fields = [] + ordering_map = { + "id": ("id", False), + "email": ("email", False), + "username": ("username", False), + "phone": ("phone", False), + "is_active": ("is_active", False), + "is_verified": ("is_verified", False), + "created_at": ("created_at", False), + "updated_at": ("updated_at", False), + "first_name": ("profile__first_name", True), + "middle_name": ("profile__mid_name", True), + "last_name": ("profile__last_name", True), + "role": ("is_staff", False), + } + + for raw_field in (item.strip() for item in ordering.split(",") if item.strip()): + is_desc = raw_field.startswith("-") + field_name = raw_field[1:] if is_desc else raw_field + mapped_config = ordering_map.get(field_name) + if not mapped_config: + continue + mapped_field, nulls_last = mapped_config + if nulls_last: + ordering_fields.append( + F(mapped_field).desc(nulls_last=True) + if is_desc + else F(mapped_field).asc(nulls_last=True) + ) + continue + ordering_fields.append(f"-{mapped_field}" if is_desc else mapped_field) + + if ordering_fields: + queryset = queryset.order_by(*ordering_fields, "-created_at") + + return queryset + @classmethod def update_user(cls, user_id: int, **fields) -> User: """ @@ -107,6 +185,79 @@ class UserService: user.save() return user + @classmethod + @transaction.atomic + def create_managed_user( + cls, + *, + email: str, + username: str, + password: str, + role: str, + first_name: str, + last_name: str, + middle_name: str | None = None, + **extra_fields, + ) -> User: + """Создаёт пользователя администратором и назначает ему роль.""" + user = User.objects.create_user( + email=email, + username=username, + password=password, + **extra_fields, + ) + cls.assign_role(user, role) + cls._update_or_create_profile( + user=user, + first_name=first_name, + middle_name=middle_name or "", + last_name=last_name, + ) + return cls.get_users_queryset().get(id=user.id) + + @classmethod + @transaction.atomic + def update_managed_user(cls, user_id: int, **fields) -> User: + """Обновляет учётные данные, профиль, пароль и роль пользователя.""" + user = cls.get_user_by_id(user_id) + role = fields.pop("role", None) + password = fields.pop("password", None) + profile_fields = { + key: fields.pop(key) + for key in ("first_name", "middle_name", "last_name") + if key in fields + } + + for field, value in fields.items(): + setattr(user, field, value) + + if password: + user.set_password(password) + user.save() + + if role is not None: + cls.assign_role(user, role) + if profile_fields: + cls._update_or_create_profile(user=user, **profile_fields) + + return cls.get_users_queryset().get(id=user.id) + + @classmethod + def deactivate_user(cls, user_id: int) -> User: + """Деактивирует пользователя без удаления записи.""" + user = cls.get_user_by_id(user_id) + user.is_active = False + user.save(update_fields=["is_active"]) + return user + + @classmethod + def activate_user(cls, user_id: int) -> User: + """Возвращает пользователя в активное состояние.""" + user = cls.get_user_by_id(user_id) + user.is_active = True + user.save(update_fields=["is_active"]) + return user + @classmethod def delete_user(cls, user_id: int) -> None: """ @@ -138,6 +289,74 @@ class UserService: "access": str(refresh.access_token), } + @classmethod + def ensure_role_groups(cls) -> dict[str, Group]: + """Гарантирует наличие системных групп ролей.""" + groups: dict[str, Group] = {} + for role, _label in cls.ROLE_CHOICES: + group, _ = Group.objects.get_or_create(name=role) + groups[role] = group + return groups + + @classmethod + def get_user_role(cls, user: User) -> str: + """Возвращает прикладную роль пользователя.""" + if user.is_superuser or user.is_staff: + return cls.ROLE_ADMIN + group_names = {group.name for group in user.groups.all()} + if cls.ROLE_ADMIN in group_names: + return cls.ROLE_ADMIN + return cls.ROLE_USER + + @classmethod + def get_role_label(cls, role: str) -> str: + """Возвращает человекочитаемое название роли.""" + return cls.ROLE_LABELS.get(role, role) + + @classmethod + def get_user_capabilities(cls, user: User) -> dict[str, bool]: + """Возвращает доступные пользователю административные возможности.""" + return { + "can_access_admin_page": cls.get_user_role(user) == cls.ROLE_ADMIN, + } + + @classmethod + def assign_role(cls, user: User, role: str) -> User: + """Назначает роль через auth.Group и согласует staff-флаги.""" + if role not in cls.ROLE_LABELS: + raise ValueError(f"Unsupported role: {role}") + + groups = cls.ensure_role_groups() + current_role_groups = list(user.groups.filter(name__in=groups)) + if current_role_groups: + user.groups.remove(*current_role_groups) + user.groups.add(groups[role]) + + user.is_staff = role == cls.ROLE_ADMIN + if role == cls.ROLE_USER: + user.is_superuser = False + user.save(update_fields=["is_staff", "is_superuser"]) + return user + + @classmethod + def _update_or_create_profile( + cls, + *, + user: User, + first_name: str | None = None, + middle_name: str | None = None, + last_name: str | None = None, + ) -> Profile: + profile, _ = Profile.objects.get_or_create(user=user) + if first_name is not None: + profile.first_name = first_name + if middle_name is not None: + profile.mid_name = middle_name + if last_name is not None: + profile.last_name = last_name + profile.save() + return profile + @classmethod def verify_email(cls, user_id: int) -> User: """ diff --git a/src/apps/user/urls.py b/src/apps/user/urls.py index c25dd3b..2e47a17 100644 --- a/src/apps/user/urls.py +++ b/src/apps/user/urls.py @@ -15,6 +15,21 @@ urlpatterns = [ # Пользовательские данные path("me/", views.CurrentUserView.as_view(), name="current_user"), path("admin/users/", views.AdminUsersManagementView.as_view(), name="admin_users"), + path( + "admin/users//", + views.AdminUserDetailView.as_view(), + name="admin-user-detail", + ), + path( + "admin/users//deactivate/", + views.AdminUserDeactivateView.as_view(), + name="admin-user-deactivate", + ), + path( + "admin/users//activate/", + views.AdminUserActivateView.as_view(), + name="admin-user-activate", + ), path("me/update/", views.UserUpdateView.as_view(), name="user_update"), path("profile/", views.ProfileDetailView.as_view(), name="profile_detail"), path("profile/full/", views.user_profile_detail, name="profile_full"), diff --git a/src/apps/user/views.py b/src/apps/user/views.py index 6b7d11e..18dbd06 100644 --- a/src/apps/user/views.py +++ b/src/apps/user/views.py @@ -1,13 +1,18 @@ +from urllib.parse import urlencode + from apps.core.models import BackgroundJob from apps.core.services import BackgroundJobService from django.contrib.auth import authenticate, get_user_model from django.contrib.auth.hashers import check_password +from django.core.paginator import Paginator from django.db.models import F from django.db.models.functions import Coalesce +from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from rest_framework import generics, status from rest_framework.decorators import api_view, permission_classes +from rest_framework.exceptions import ValidationError from rest_framework.permissions import AllowAny, IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.views import APIView @@ -15,8 +20,12 @@ from rest_framework_simplejwt.exceptions import TokenError from rest_framework_simplejwt.tokens import RefreshToken from .serializers import ( + AdminUserCreateSerializer, + AdminUserListResponseSerializer, + AdminUserUpdateSerializer, CurrentUserSerializer, LoginSerializer, + ManagedUserSerializer, PasswordChangeSerializer, ProfileUpdateSerializer, TokenSerializer, @@ -34,6 +43,56 @@ AUTH_TAG = "Аутентификация" USER_TAG = "Пользователь" +def _build_page_url(request, page_number: int) -> str: + query_params = request.query_params.copy() + query_params["page"] = page_number + encoded_query = urlencode(query_params, doseq=True) + return request.build_absolute_uri(f"{request.path}?{encoded_query}") + + +def _paginate_user_queryset(request, queryset): + """Пагинирует только запросы с явными page/page_size для legacy-совместимости.""" + if "page" not in request.query_params and "page_size" not in request.query_params: + return { + "count": queryset.count(), + "next": None, + "previous": None, + "results": list(queryset), + } + + page_size_raw = request.query_params.get("page_size", "20") + page_raw = request.query_params.get("page", "1") + try: + page_size = max(1, min(int(page_size_raw), 100)) + page_number = max(1, int(page_raw)) + except (TypeError, ValueError) as exc: + raise ValidationError( + { + "detail": ( + "Параметры page и page_size должны быть положительными " + "целыми числами." + ) + } + ) from exc + + paginator = Paginator(queryset, page_size) + page_obj = paginator.get_page(page_number) + return { + "count": paginator.count, + "next": ( + _build_page_url(request, page_obj.next_page_number()) + if page_obj.has_next() + else None + ), + "previous": ( + _build_page_url(request, page_obj.previous_page_number()) + if page_obj.has_previous() + else None + ), + "results": list(page_obj.object_list), + } + + class RegisterView(APIView): """ Регистрация нового пользователя. @@ -239,7 +298,7 @@ class PasswordChangeView(APIView): class AdminUsersManagementView(APIView): - """Список пользователей для административной страницы управления.""" + """Список и создание пользователей в административной панели.""" permission_classes = [IsAdminUser] @@ -276,15 +335,152 @@ class AdminUsersManagementView(APIView): operation_summary="Список пользователей (admin)", operation_description=( "Возвращает пользователей для административной панели управления. " - "Включает метрики последней фоновой задачи пользователя." + "Включает роль, профиль и метрики последней фоновой задачи пользователя." ), - responses={200: UserManagementSerializer(many=True)}, + manual_parameters=[ + openapi.Parameter( + "search", + openapi.IN_QUERY, + type=openapi.TYPE_STRING, + required=False, + ), + openapi.Parameter( + "ordering", + openapi.IN_QUERY, + type=openapi.TYPE_STRING, + required=False, + ), + openapi.Parameter( + "page", + openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + ), + openapi.Parameter( + "page_size", + openapi.IN_QUERY, + type=openapi.TYPE_INTEGER, + required=False, + ), + ], + responses={200: AdminUserListResponseSerializer}, ) def get(self, request): - users = User.objects.all().select_related("profile").order_by("id") - self._attach_latest_jobs(list(users)) + queryset = UserService.get_filtered_users_queryset( + search=request.query_params.get("search", ""), + ordering=request.query_params.get("ordering", ""), + ) + paginated = _paginate_user_queryset(request, queryset) + users = paginated["results"] + self._attach_latest_jobs(users) serializer = UserManagementSerializer(users, many=True) - return Response({"results": serializer.data}) + return Response( + { + "count": paginated["count"], + "next": paginated["next"], + "previous": paginated["previous"], + "results": serializer.data, + } + ) + + @swagger_auto_schema( + tags=[USER_TAG], + operation_summary="Создать пользователя (admin)", + request_body=AdminUserCreateSerializer, + responses={201: ManagedUserSerializer}, + ) + def post(self, request): + serializer = AdminUserCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + user = UserService.create_managed_user(**serializer.validated_data) + return Response( + ManagedUserSerializer(user).data, + status=status.HTTP_201_CREATED, + ) + + +class AdminUserDetailView(APIView): + """Просмотр и частичное обновление пользователя администратором.""" + + permission_classes = [IsAdminUser] + + @staticmethod + def _get_user(user_id: int): + return get_object_or_404(UserService.get_users_queryset(), id=user_id) + + @swagger_auto_schema( + tags=[USER_TAG], + operation_summary="Детали пользователя (admin)", + responses={200: UserManagementSerializer}, + ) + def get(self, request, user_id: int): + user = self._get_user(user_id) + AdminUsersManagementView._attach_latest_jobs([user]) + return Response(UserManagementSerializer(user).data) + + @swagger_auto_schema( + tags=[USER_TAG], + operation_summary="Обновить пользователя (admin)", + request_body=AdminUserUpdateSerializer, + responses={200: ManagedUserSerializer}, + ) + def patch(self, request, user_id: int): + user = self._get_user(user_id) + serializer = AdminUserUpdateSerializer(user, data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + + if request.user.id == user.id: + if serializer.validated_data.get("is_active") is False: + return Response( + {"detail": "Нельзя деактивировать самого себя."}, + status=status.HTTP_400_BAD_REQUEST, + ) + if serializer.validated_data.get("role") == UserService.ROLE_USER: + return Response( + {"detail": "Нельзя снять у себя роль администратора."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + updated_user = UserService.update_managed_user( + user_id=user.id, + **serializer.validated_data, + ) + return Response(ManagedUserSerializer(updated_user).data) + + +class AdminUserDeactivateView(APIView): + """Деактивация пользователя администратором.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[USER_TAG], + operation_summary="Деактивировать пользователя (admin)", + responses={200: ManagedUserSerializer}, + ) + def post(self, request, user_id: int): + if request.user.id == user_id: + return Response( + {"detail": "Нельзя деактивировать самого себя."}, + status=status.HTTP_400_BAD_REQUEST, + ) + user = UserService.deactivate_user(user_id) + return Response(ManagedUserSerializer(user).data) + + +class AdminUserActivateView(APIView): + """Активация пользователя администратором.""" + + permission_classes = [IsAdminUser] + + @swagger_auto_schema( + tags=[USER_TAG], + operation_summary="Активировать пользователя (admin)", + responses={200: ManagedUserSerializer}, + ) + def post(self, request, user_id: int): + user = UserService.activate_user(user_id) + return Response(ManagedUserSerializer(user).data) @swagger_auto_schema( diff --git a/src/core/api_v2_urls.py b/src/core/api_v2_urls.py new file mode 100644 index 0000000..792da40 --- /dev/null +++ b/src/core/api_v2_urls.py @@ -0,0 +1,12 @@ +"""API v2 routes shared with the Mostovik administrative frontend contract.""" + +from django.urls import include, path + +app_name = "api_v2" + +urlpatterns = [ + path( + "organization-source-records/", + include("apps.external_data.export_urls"), + ), +] diff --git a/src/core/urls.py b/src/core/urls.py index 4e71e67..9dfcd68 100644 --- a/src/core/urls.py +++ b/src/core/urls.py @@ -40,6 +40,7 @@ urlpatterns = [ path("admin/", admin.site.urls), path("health/", include("apps.core.urls")), path("api/v1/", include("core.api_v1_urls", namespace="api_v1")), + path("api/v2/", include("core.api_v2_urls", namespace="api_v2")), path("auth/", include("rest_framework.urls")), ] diff --git a/src/settings/base.py b/src/settings/base.py index 2599a80..7344c2b 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -231,6 +231,26 @@ STATICFILES_STORAGE = "whitenoise.storage.CompressedManifestStaticFilesStorage" MEDIA_URL = "/media/" MEDIA_ROOT = PROJECT_ROOT / "media" +SOURCE_RECORD_EXPORT_DIRECTORY = os.getenv( + "SOURCE_RECORD_EXPORT_DIRECTORY", + str(PROJECT_ROOT / "media" / "source-record-exports"), +) +SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP = int( + os.getenv("SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP", "2") +) +SOURCE_RECORD_EXPORT_LOCK_KEY = os.getenv( + "SOURCE_RECORD_EXPORT_LOCK_KEY", + "external-data:source-record-exports:lock", +) +SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS = int( + os.getenv("SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", str(6 * 60 * 60)) +) +SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE = int( + os.getenv("SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE", "100000") +) +SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS = int( + os.getenv("SOURCE_RECORD_EXPORT_DOWNLOAD_TICKET_TTL_SECONDS", "300") +) DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" AUTH_USER_MODEL = "user.User" diff --git a/tests/apps/core/test_demo_period_workbooks.py b/tests/apps/core/test_demo_period_workbooks.py new file mode 100644 index 0000000..b0c90ef --- /dev/null +++ b/tests/apps/core/test_demo_period_workbooks.py @@ -0,0 +1,85 @@ +"""Integration checks for the versioned Excel demo-period fixtures.""" + +import re +from pathlib import Path + +from apps.form_1.models import FormF1Record +from apps.form_1.services import FormF1Parser +from apps.form_3.models import FormF3Record +from apps.form_3.services import FormF3Parser +from apps.form_4.services import FormF4Parser +from apps.form_6.services import FormF6Parser +from django.test import TestCase + +DEMO_DIR = Path(__file__).resolve().parents[3] / "input" / "demo-periods" + + +class DemoPeriodWorkbooksTest(TestCase): + def test_form_f3_workbooks_match_current_schema(self): + for path in sorted(DEMO_DIR.glob("Ф-3_*.xlsx")): + year = int(re.search(r"_(\d{4})", path.name).group(1)) + f1_path = max( + DEMO_DIR.glob(f"Ф-1_{year}-*.xlsx"), + key=lambda item: int(re.search(r"-(\d{2})", item.name).group(1)), + ) + report_month = int(re.search(r"-(\d{2})", f1_path.name).group(1)) + with ( + self.subTest(path=path.name), + f1_path.open("rb") as f1_file, + path.open("rb") as f3_file, + ): + f1_result = FormF1Parser( + report_year=year, + report_month=report_month, + ).parse(f1_file) + f3_result = FormF3Parser(report_year=year).parse(f3_file) + self.assertEqual(f1_result.loaded_count, 20) + self.assertEqual(f1_result.skipped_count, 0) + self.assertEqual(f3_result.loaded_count, 20) + self.assertEqual(f3_result.skipped_count, 0) + + for f3_record in FormF3Record.objects.filter( + load_batch=f3_result.batch_id + ).select_related("organization"): + f1_record = FormF1Record.objects.get( + organization=f3_record.organization, + load_batch=f1_result.batch_id, + ) + age_group_total = sum( + ( + f3_record.employees_under_20, + f3_record.employees_20_29, + f3_record.employees_30_39, + f3_record.employees_40_49, + f3_record.employees_50_59, + f3_record.employees_over_60, + ) + ) + self.assertEqual( + age_group_total, + int(f1_record.avg_employees), + msg=( + f"{path.name}: возрастные группы не совпадают " + f"с {f1_path.name} для ИНН {f3_record.organization.inn}" + ), + ) + + def test_form_f4_workbooks_match_current_schema(self): + for path in sorted(DEMO_DIR.glob("Ф-4_*.xlsx")): + match = re.search(r"_(\d{4})-H([12])", path.name) + year, half_year = map(int, match.groups()) + with self.subTest(path=path.name), path.open("rb") as file: + result = FormF4Parser( + report_year=year, + report_half_year=half_year, + ).parse(file) + self.assertEqual(result.loaded_count, 20) + self.assertEqual(result.skipped_count, 0) + + def test_form_f6_workbooks_are_structurally_consistent(self): + for path in sorted(DEMO_DIR.glob("Ф-6_*.xlsx")): + year = int(re.search(r"_(\d{4})", path.name).group(1)) + with self.subTest(path=path.name), path.open("rb") as file: + result = FormF6Parser(report_year=year).parse(file) + self.assertEqual(result.loaded_count, 20) + self.assertEqual(result.skipped_count, 0) diff --git a/tests/apps/core/test_generate_test_reports_command.py b/tests/apps/core/test_generate_test_reports_command.py index 828ac17..995af62 100644 --- a/tests/apps/core/test_generate_test_reports_command.py +++ b/tests/apps/core/test_generate_test_reports_command.py @@ -59,6 +59,51 @@ class GenerateTestReportsCommandTest(TestCase): self.assertEqual(PublicProcurement.objects.count(), 3) self.assertEqual(ArbitrationCase.objects.count(), 3) + for record in FormF3Record.objects.all(): + self.assertEqual( + sum( + ( + record.employees_under_20, + record.employees_20_29, + record.employees_30_39, + record.employees_40_49, + record.employees_50_59, + record.employees_over_60, + ) + ), + int(record.avg_employees), + ) + self.assertEqual( + record.machine_tools_and_equipment, + record.total_equipment, + ) + + for record in FormF6Record.objects.all(): + self.assertEqual( + sum( + ( + record.age_under_5, + record.age_5_10, + record.age_10_15, + record.age_15_20, + record.age_over_20, + ) + ), + record.total_equipment, + ) + self.assertEqual( + sum( + ( + record.age_under_5_imported, + record.age_5_10_imported, + record.age_10_15_imported, + record.age_15_20_imported, + record.age_over_20_imported, + ) + ), + record.imported_equipment, + ) + self.assertIn( "Ф-1: создано 12 записей, активных 9, архивных 3", stdout.getvalue() ) diff --git a/tests/apps/core/test_restore_report_upload_files_command.py b/tests/apps/core/test_restore_report_upload_files_command.py new file mode 100644 index 0000000..0e7bdd8 --- /dev/null +++ b/tests/apps/core/test_restore_report_upload_files_command.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import hashlib +import tempfile +from pathlib import Path + +from apps.core.models import ReportUpload, ReportUploadStatus +from django.core.management import call_command +from django.test import TestCase, override_settings + + +class RestoreReportUploadFilesCommandTest(TestCase): + def _upload(self, *, content: bytes) -> ReportUpload: + digest = hashlib.sha256(content).hexdigest() + upload = ReportUpload.objects.create( + form="f1", + load_batch=101, + original_file="report_uploads/f1/restored/report.xlsx", + file_name="report.xlsx", + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + file_size=len(content), + file_hash=digest, + status=ReportUploadStatus.SUCCESS, + ) + return upload + + def test_dry_run_then_restore_and_repeat(self): + content = b"exact original workbook bytes" + upload = self._upload(content=content) + + with tempfile.TemporaryDirectory() as media_dir, tempfile.TemporaryDirectory() as source_dir: + source_path = Path(source_dir) / "original.xlsx" + source_path.write_bytes(content) + with override_settings(MEDIA_ROOT=media_dir): + dry_run_result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + dry_run=True, + ) + self.assertIn('"restored": 1', dry_run_result) + self.assertFalse( + upload.original_file.storage.exists(upload.original_file.name) + ) + + restore_result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + ) + self.assertIn('"restored": 1', restore_result) + with upload.original_file.storage.open( + upload.original_file.name, "rb" + ) as handle: + self.assertEqual(handle.read(), content) + + repeat_result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + ) + self.assertIn('"already_present": 1', repeat_result) + + def test_existing_hash_mismatch_is_not_overwritten(self): + content = b"expected workbook" + upload = self._upload(content=content) + + with tempfile.TemporaryDirectory() as media_dir, tempfile.TemporaryDirectory() as source_dir: + (Path(source_dir) / "original.xlsx").write_bytes(content) + target_path = Path(media_dir) / upload.original_file.name + target_path.parent.mkdir(parents=True) + target_path.write_bytes(b"different bytes") + + with override_settings(MEDIA_ROOT=media_dir): + result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + ) + + self.assertIn('"hash_mismatch": 1', result) + self.assertEqual(target_path.read_bytes(), b"different bytes") diff --git a/tests/apps/exchange/test_api.py b/tests/apps/exchange/test_api.py index 6a6d7c0..d8676a3 100644 --- a/tests/apps/exchange/test_api.py +++ b/tests/apps/exchange/test_api.py @@ -316,6 +316,7 @@ def build_exchange_payload() -> dict[str, list[dict[str, object]]]: "party_role": "ответчик", "status": "in_progress", "decision_date": "2026-03-25", + "claim_amount": "1250000.50", } ], "bankruptcy_procedures": [ @@ -463,6 +464,10 @@ class ExchangePackageApiTest(APITestCase): self.assertEqual(FinancialReport.objects.count(), 1) self.assertEqual(FinancialReportLine.objects.count(), 1) self.assertEqual(ArbitrationCase.objects.count(), 1) + self.assertEqual( + ArbitrationCase.objects.get().claim_amount, + Decimal("1250000.50"), + ) self.assertEqual(BankruptcyProcedure.objects.count(), 1) self.assertEqual( BankruptcyProcedure._meta.get_field("status").max_length, @@ -484,6 +489,7 @@ class ExchangePackageApiTest(APITestCase): response.data["result"]["financial_reports"]["created_lines"], 1, ) + self.assertEqual( response.data["result"]["defense_unreliable_suppliers"]["created"], 1, @@ -530,6 +536,25 @@ class ExchangePackageApiTest(APITestCase): self.assertEqual(package_import.delivery_channel, ExchangeDeliveryChannel.API) self.assertEqual(package_import.status, "success") + def test_upload_preserves_long_prosecutor_control_authority(self): + payload = build_exchange_payload() + long_authority = "Межрегиональное контрольное управление " + "А" * 300 + payload["prosecutor_checks"][0]["control_authority"] = long_authority + archive = build_exchange_archive(data=payload) + + response = self.client.post( + self.url, + {"file": archive}, + format="multipart", + HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN, + ) + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual( + ProsecutorCheck.objects.get().control_authority, + long_authority, + ) + def test_upload_rejects_invalid_exchange_token(self): archive = build_exchange_archive(data=build_exchange_payload()) invalid_token = get_random_string(24) @@ -744,6 +769,51 @@ class ExchangePackageApiTest(APITestCase): duplicate_import = ExchangePackageImport.objects.order_by("-created_at").first() self.assertIsNotNone(duplicate_import.duplicate_of) + def test_upload_replaces_stale_financial_report_lines(self): + first_payload = build_exchange_payload() + first_archive = build_exchange_archive( + package_id="pkg-financial-year-2025", + data=first_payload, + ) + first_response = self.client.post( + self.url, + {"file": first_archive}, + format="multipart", + HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN, + ) + self.assertEqual(first_response.status_code, status.HTTP_201_CREATED) + self.assertEqual( + list(FinancialReportLine.objects.values_list("year", flat=True)), + [2025], + ) + + second_payload = build_exchange_payload() + second_payload["financial_reports"][0]["lines"][0]["year"] = 2026 + second_archive = build_exchange_archive( + package_id="pkg-financial-year-2026", + data=second_payload, + ) + second_response = self.client.post( + self.url, + {"file": second_archive}, + format="multipart", + HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN, + ) + + self.assertEqual(second_response.status_code, status.HTTP_201_CREATED) + self.assertEqual( + list(FinancialReportLine.objects.values_list("year", flat=True)), + [2026], + ) + self.assertEqual( + second_response.data["result"]["financial_reports"]["created_lines"], + 1, + ) + self.assertEqual( + second_response.data["result"]["financial_reports"]["deleted_lines"], + 1, + ) + def test_cli_import_uses_same_pipeline(self): archive = build_exchange_archive( package_id="pkg-cli-001", diff --git a/tests/apps/external_data/factories.py b/tests/apps/external_data/factories.py index f6619b1..1089ad8 100644 --- a/tests/apps/external_data/factories.py +++ b/tests/apps/external_data/factories.py @@ -101,6 +101,9 @@ class ArbitrationCaseFactory(factory.django.DjangoModelFactory): court_name = "Арбитражный суд города Москвы" party_role = "defendant" status = "hearing_scheduled" + claim_amount = factory.LazyAttribute( + lambda _: fake.pydecimal(left_digits=8, right_digits=2, positive=True) + ) decision_date = factory.LazyAttribute(lambda _: fake.date_this_year()) diff --git a/tests/apps/external_data/test_api.py b/tests/apps/external_data/test_api.py index 6aa2ec6..4202ccd 100644 --- a/tests/apps/external_data/test_api.py +++ b/tests/apps/external_data/test_api.py @@ -124,6 +124,7 @@ class ExternalDataApiTest(APITestCase): self.assertEqual(procurement_response.data["count"], 1) self.assertEqual(arbitration_response.status_code, status.HTTP_200_OK) self.assertEqual(arbitration_response.data["count"], 1) + self.assertIn("claim_amount", arbitration_response.data["results"][0]) def test_corporation_memberships_filter(self): InformationSecurityRegistryEntryFactory( diff --git a/tests/apps/external_data/test_export_tasks.py b/tests/apps/external_data/test_export_tasks.py new file mode 100644 index 0000000..036d9e7 --- /dev/null +++ b/tests/apps/external_data/test_export_tasks.py @@ -0,0 +1,72 @@ +"""Tests for the external-data export task and schedule.""" + +from importlib import import_module +from tempfile import TemporaryDirectory + +from apps.external_data.tasks import refresh_source_record_export_artifacts +from django.apps import apps as django_apps +from django.conf import settings +from django.core.cache import cache +from django.test import TestCase, override_settings +from django.utils import timezone +from django_celery_beat.models import PeriodicTask + + +class SourceRecordExportArtifactsTaskTest(TestCase): + """Check nightly artifact generation and its distributed lock.""" + + def setUp(self): + cache.clear() + self.export_directory = TemporaryDirectory() + self.settings_override = override_settings( + SOURCE_RECORD_EXPORT_DIRECTORY=self.export_directory.name, + SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP=2, + SOURCE_RECORD_EXPORT_LOCK_KEY="test:state-corp-source-exports:lock", + SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS=300, + ) + self.settings_override.enable() + + def tearDown(self): + self.settings_override.disable() + self.export_directory.cleanup() + cache.clear() + super().tearDown() + + def test_refresh_task_builds_matrix_and_releases_lock(self): + result = refresh_source_record_export_artifacts() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["artifacts_count"], 25) + self.assertEqual(result["export_year"], timezone.localdate().year) + self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY)) + + def test_refresh_task_skips_when_generation_lock_is_held(self): + cache.set(settings.SOURCE_RECORD_EXPORT_LOCK_KEY, "busy", timeout=300) + + result = refresh_source_record_export_artifacts() + + self.assertEqual(result, {"status": "skipped", "reason": "locked"}) + + +class SourceRecordExportScheduleMigrationTest(TestCase): + """Check the nightly Celery Beat schedule for prepared exports.""" + + def test_migration_seeds_nightly_export_task_idempotently(self): + migration = import_module( + "apps.external_data.migrations.0007_seed_nightly_source_record_exports" + ) + + migration.seed_nightly_source_record_export_schedule(django_apps, None) + migration.seed_nightly_source_record_export_schedule(django_apps, None) + + task = PeriodicTask.objects.get(name=migration.NIGHTLY_SOURCE_EXPORT_TASK_NAME) + self.assertEqual( + task.task, + "apps.external_data.tasks.refresh_source_record_export_artifacts", + ) + self.assertTrue(task.enabled) + self.assertEqual(task.args, "[]") + self.assertEqual(task.kwargs, "{}") + self.assertEqual(task.crontab.minute, "30") + self.assertEqual(task.crontab.hour, "5") + self.assertEqual(str(task.crontab.timezone), "Europe/Moscow") diff --git a/tests/apps/external_data/test_source_record_export.py b/tests/apps/external_data/test_source_record_export.py new file mode 100644 index 0000000..c04aa32 --- /dev/null +++ b/tests/apps/external_data/test_source_record_export.py @@ -0,0 +1,479 @@ +"""Tests for prepared State Corp external-data exports.""" + +import json +import zipfile +from datetime import UTC, datetime +from io import BytesIO, StringIO +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from apps.external_data.source_record_export import ( + ORGANIZATION_EXPORT_FIELDS, + SOURCE_GROUP_EXPORT_SPECS, + SOURCE_RECORD_EXPORT_FIELDS, + SourceRecordExportArtifactsUnavailable, + _source_group_headers, + build_source_record_export_artifacts, + build_source_records_export_archive, + load_current_source_record_export_generation, +) +from django.core.management import call_command +from django.test import override_settings +from django.utils import timezone +from openpyxl import load_workbook +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.external_data.factories import ( + ArbitrationCaseFactory, + BankruptcyProcedureFactory, + FinancialReportFactory, + FinancialReportLineFactory, + IndustrialCertificateFactory, + IndustrialProductFactory, + ManufacturerRegistryEntryFactory, + ProsecutorCheckFactory, + PublicProcurementFactory, +) +from tests.apps.organization.factories import OrganizationFactory +from tests.apps.user.factories import UserFactory + + +class SourceRecordExportApiTest(APITestCase): + """Check admin access and zero-query delivery of prepared files.""" + + export_url = "/api/v2/organization-source-records/export/" + ticket_url = "/api/v2/organization-source-records/export-ticket/" + download_url = "/api/v2/organization-source-records/export-download/" + + def setUp(self): + self.export_directory = TemporaryDirectory() + self.settings_override = override_settings( + SOURCE_RECORD_EXPORT_DIRECTORY=self.export_directory.name, + SOURCE_RECORD_EXPORT_GENERATIONS_TO_KEEP=2, + ) + self.settings_override.enable() + + def tearDown(self): + self.settings_override.disable() + self.export_directory.cleanup() + super().tearDown() + + @staticmethod + def _response_body(response) -> bytes: + return b"".join(response.streaming_content) + + def test_export_is_unavailable_before_first_generation(self): + self.client.force_authenticate(UserFactory.create_superuser()) + + response = self.client.post( + self.export_url, + {"sources": ["planned_inspections"], "format": "json"}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_503_SERVICE_UNAVAILABLE) + self.assertEqual(response.data["code"], "source_export_not_ready") + self.assertEqual(response["Retry-After"], "3600") + + def test_generation_builds_full_matrix_from_normalized_tables(self): + current_date = timezone.localdate() + organization = OrganizationFactory.create( + full_name='Акционерное общество "Экспорт"', + okpo="12345678", + ) + IndustrialProductFactory.create(organization=organization) + IndustrialCertificateFactory.create( + organization=organization, + issue_date=current_date, + ) + ManufacturerRegistryEntryFactory.create(organization=organization) + ProsecutorCheckFactory.create( + organization=organization, + start_date=current_date, + ) + arbitration_case = ArbitrationCaseFactory.create( + organization=organization, + decision_date=current_date, + ) + report = FinancialReportFactory.create(organization=organization) + FinancialReportLineFactory.create( + report=report, + line_code="1600", + year=current_date.year, + ) + + generation = build_source_record_export_artifacts() + + self.assertEqual(generation.artifacts_count, 25) + self.assertEqual(generation.files_count, 25) + self.assertEqual(generation.records_count, 6) + self.assertEqual(generation.export_year, current_date.year) + expected_prefix = [*ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS] + for source_spec in SOURCE_GROUP_EXPORT_SPECS.values(): + self.assertEqual( + _source_group_headers(source_spec)[: len(expected_prefix)], + expected_prefix, + ) + industrial_path = next( + artifact.path + for artifact in generation.artifacts + if artifact.source_group == "industrial_production" + and artifact.file_format == "json" + ) + industrial_rows = json.loads(industrial_path.read_text(encoding="utf-8")) + self.assertEqual( + {row["record_type"] for row in industrial_rows}, + { + "industrial_certificate", + "industrial_product", + "manufacturer_registry_entry", + }, + ) + self.assertEqual({row["ОКПО"] for row in industrial_rows}, {"12345678"}) + self.assertEqual( + list(industrial_rows[0])[: len(expected_prefix)], + expected_prefix, + ) + self.assertNotIn("organization", industrial_rows[0]) + self.assertIn("payload.product_name", industrial_rows[0]) + + financial_path = next( + artifact.path + for artifact in generation.artifacts + if artifact.source_group == "financial_indicators" + ) + financial_rows = json.loads(financial_path.read_text(encoding="utf-8")) + self.assertEqual(financial_rows[0]["financial_lines"][0]["line_code"], "1600") + + arbitration_path = next( + artifact.path + for artifact in generation.artifacts + if artifact.source_group == "arbitration" and artifact.file_format == "json" + ) + arbitration_row = json.loads(arbitration_path.read_text(encoding="utf-8"))[0] + self.assertEqual(arbitration_row["source"], "arbitration") + self.assertEqual( + arbitration_row["external_id"], arbitration_row["payload.case_number"] + ) + self.assertEqual( + arbitration_row["title"], + f"Дело {arbitration_row['payload.case_number']}", + ) + self.assertEqual( + arbitration_row["record_date"], arbitration_case.decision_date.isoformat() + ) + self.assertEqual(arbitration_row["status"], arbitration_row["payload.status"]) + self.assertEqual( + arbitration_row["amount"], + str(arbitration_case.claim_amount), + ) + self.assertEqual( + arbitration_row["payload.claim_amount"], + str(arbitration_case.claim_amount), + ) + self.assertEqual(arbitration_row["payload.role"], "defendant") + self.assertIn("payload.court", arbitration_row) + self.assertEqual(arbitration_row["payload.source"], "arbitration") + self.assertEqual(arbitration_row["payload.inn"], organization.inn) + self.assertEqual( + [key for key in arbitration_row if key.startswith("payload.")], + [ + "payload.inn", + "payload.url", + "payload.ogrn", + "payload.role", + "payload.court", + "payload.source", + "payload.status", + "payload.load_batch", + "payload.case_number", + "payload.claim_amount", + "payload.organisation_name", + ], + ) + + def test_generation_keeps_provider_records_but_hides_provider_mentions(self): + organization = OrganizationFactory.create(okpo="11223344") + excluded_record = BankruptcyProcedureFactory.create( + organization=organization, + external_id="checko-fedresurs:123", + source_url="https://api.checko.ru/v2/bankruptcy/123", + ) + included_record = BankruptcyProcedureFactory.create( + organization=organization, + external_id="checkout-reference:456", + source_url="https://fedresurs.ru/message/456", + ) + + generation = build_source_record_export_artifacts() + + self.assertEqual(generation.records_count, 2) + artifacts = [ + artifact + for artifact in generation.artifacts + if artifact.source_group == "bankruptcy" + ] + self.assertEqual( + {artifact.file_format for artifact in artifacts}, {"csv", "xlsx", "json"} + ) + for artifact in artifacts: + if artifact.file_format == "json": + exported_values = json.loads(artifact.path.read_text(encoding="utf-8")) + self.assertEqual( + [key for key in exported_values[0] if key.startswith("payload.")], + [ + "payload.inn", + "payload.url", + "payload.ogrn", + "payload.source", + "payload.messages", + "payload.load_batch", + "payload.case_number", + "payload.message_date", + "payload.message_type", + "payload.messages_count", + "payload.organisation_name", + ], + ) + elif artifact.file_format == "csv": + exported_values = artifact.path.read_text(encoding="utf-8-sig") + else: + workbook = load_workbook(artifact.path, read_only=True) + exported_values = list(workbook["data"].iter_rows(values_only=True)) + serialized_values = str(exported_values) + self.assertIn(str(excluded_record.id), serialized_values) + self.assertIn(str(included_record.id), str(exported_values)) + self.assertIn("11223344", str(exported_values)) + self.assertNotRegex( + serialized_values, + r"(?i)(? BytesIO: + headers = [ + "Наименование организации", + "ОКПО", + "ОГРН", + "ИНН", + "Отгруженные товары собственного производства, выполненные работы и услуги собственными силами в фактических ценах - всего", + "Отгруженные товары собственного производства, выполненные работы и услуги собственными силами в фактических ценах, из них инновационные товары, работы, услуги", + "Всего основных фондов (без незавершенных активов и не включая земельные участки и объекты природопользования)", + "Станочный парк и оборудование, всего по предприятию", + "Станочный парк и оборудование - импортное оборудование", + "Оборудование с возрастом от 5 до 10 лет", + "Оборудование с возрастом до 5 лет", + "Фактический (физический) износ оборудования, %", + "Средний уровень загрузки производственных мощностей (всего)", + "Средний уровень загрузки производственных мощностей военного производства", + "Средний возраст сотрудников", + "Количество работников предприятия до 20 лет", + "Количество работников предприятия 20-29 лет", + "Количество работников предприятия 30-39 лет", + "Количество работников предприятия 40-49 лет", + "Количество работников предприятия 50-59 лет", + "Количество работников предприятия старше 60 лет", + ] + workbook = Workbook() + sheet = workbook.active + sheet.append(headers) + sheet.append( + [None, None, None, None] + + ["тыс. руб."] * 3 + + ["ед."] * 4 + + ["%"] * 3 + + ["лет"] + + ["чел."] * 6 + ) + sheet.append( + [ + "Тестовая организация Ф-3", + "90000001", + "1267700000017", + "3234567890", + 352, + 222, + 85, + 46, + 17, + 13, + 14, + 21, + 79, + 65, + 42, + 4, + 18, + 28, + 26, + 17, + 7, + ] + ) + file = BytesIO() + workbook.save(file) + file.seek(0) + return file + def test_get_column_mappings_returns_mappings(self): """Test get_column_mappings returns correct mappings.""" parser = FormF3Parser(report_year=2026, report_quarter=1) @@ -57,8 +128,24 @@ class FormF3ParserTest(TestCase): self.assertTrue(len(mappings) > 0) field_names = [m.field_name for m in mappings] - self.assertIn("avg_employees", field_names) - self.assertIn("total_equipment", field_names) + self.assertIn("shipped_goods_total", field_names) + self.assertIn("machine_tools_and_equipment", field_names) + self.assertIn("average_employee_age", field_names) + self.assertNotIn("avg_employees", field_names) + + def test_parse_current_template_maps_equipment_and_personnel_columns(self): + OrganizationFactory.create(inn="3234567890") + parser = FormF3Parser(report_year=2026, report_quarter=1) + + result = parser.parse(self._current_template_file()) + + self.assertEqual(result.loaded_count, 1) + record = FormF3Record.objects.get() + self.assertEqual(record.shipped_goods_total, Decimal("352")) + self.assertEqual(record.machine_tools_and_equipment, 46) + self.assertEqual(record.imported_equipment, 17) + self.assertEqual(record.average_employee_age, Decimal("42")) + self.assertEqual(record.employees_30_39, 28) def test_create_record_uses_existing_organization(self): """Report imports reuse existing organizations when available.""" @@ -69,8 +156,8 @@ class FormF3ParserTest(TestCase): row_data = { "inn": "3234567890", "name": "Тестовая организация Ф-3", - "avg_employees": 100, - "total_equipment": 50, + "machine_tools_and_equipment": 50, + "average_employee_age": 42, } record = parser.create_record(row_data) diff --git a/tests/apps/form_4/test_services.py b/tests/apps/form_4/test_services.py index fafe7ad..4190ee7 100644 --- a/tests/apps/form_4/test_services.py +++ b/tests/apps/form_4/test_services.py @@ -1,7 +1,12 @@ """Tests for FormF4 services.""" +from decimal import Decimal +from io import BytesIO + +from apps.form_4.models import FormF4Record from apps.form_4.services import FormF4Parser, FormF4Service from django.test import TestCase +from openpyxl import Workbook from tests.apps.organization.factories import OrganizationFactory @@ -48,6 +53,47 @@ class FormF4ServiceTest(TestCase): class FormF4ParserTest(TestCase): """Tests for FormF4Parser.""" + @staticmethod + def _current_template_file() -> BytesIO: + headers = [ + "Наименование организации", + "ОКПО", + "ОГРН", + "ИНН", + "Выручка, в соответствии с РСБУ", + "Выручка, в соответствии с МСФО", + "Чистая прибыль (убыток), в соответствии с РСБУ", + "Чистая прибыль (убыток), в соответствии с МСФО", + "Кредиты и займы, в соответствии с РСБУ", + "Кредиты и займы, в соответствии с МСФО", + "EBITDA, в соответствии с РСБУ", + "EBITDA, в соответствии с МСФО", + ] + workbook = Workbook() + sheet = workbook.active + sheet.append(headers) + sheet.append([None, None, None, None] + ["тыс. руб."] * 8) + sheet.append( + [ + "Тестовая организация Ф-4", + "90000001", + "1267700000017", + "4234567890", + 477807, + 499308, + 442414, + 462322, + 42047, + 43729, + 138564, + 62115, + ] + ) + file = BytesIO() + workbook.save(file) + file.seek(0) + return file + def test_get_column_mappings_returns_mappings(self): """Test get_column_mappings returns correct mappings.""" parser = FormF4Parser(report_year=2026, report_half_year=1) @@ -59,6 +105,20 @@ class FormF4ParserTest(TestCase): field_names = [m.field_name for m in mappings] self.assertIn("revenue_rsbu", field_names) self.assertIn("net_profit_rsbu", field_names) + self.assertIn("ebitda_rsbu", field_names) + + def test_parse_current_template_maps_profit_loans_and_ebitda_columns(self): + OrganizationFactory.create(inn="4234567890") + parser = FormF4Parser(report_year=2025, report_half_year=2) + + result = parser.parse(self._current_template_file()) + + self.assertEqual(result.loaded_count, 1) + record = FormF4Record.objects.get() + self.assertEqual(record.revenue_rsbu, Decimal("477807")) + self.assertEqual(record.net_profit_rsbu, Decimal("442414")) + self.assertEqual(record.loans_rsbu, Decimal("42047")) + self.assertEqual(record.ebitda_rsbu, Decimal("138564")) def test_create_record_uses_existing_organization(self): """Report imports reuse existing organizations when available.""" diff --git a/tests/apps/form_6/test_services.py b/tests/apps/form_6/test_services.py index fad00db..bb922ef 100644 --- a/tests/apps/form_6/test_services.py +++ b/tests/apps/form_6/test_services.py @@ -1,7 +1,12 @@ """Tests for FormF6 services.""" +from io import BytesIO + +from apps.core.excel import ExcelValidationError +from apps.form_6.models import FormF6Record from apps.form_6.services import FormF6Parser, FormF6Service from django.test import TestCase +from openpyxl import Workbook from tests.apps.organization.factories import OrganizationFactory @@ -48,6 +53,82 @@ class FormF6ServiceTest(TestCase): class FormF6ParserTest(TestCase): """Tests for FormF6Parser.""" + @staticmethod + def _current_template_file( + *, + total_equipment: int = 100, + age_under_5: int | float = 20, + total_header: str = "Всего", + ) -> BytesIO: + workbook = Workbook() + sheet = workbook.active + row_1 = [None] * 38 + row_2 = [None] * 38 + row_3 = [None] * 38 + row_4 = [None] * 38 + row_1[:6] = [ + "Наименование организации", + "ОКПО", + "ОГРН", + "ИНН", + "Код строки", + "Категория", + ] + row_1[6] = "Количество оборудования на конец года ед." + row_1[10] = "Из установленного оборудования – оборудование в возрасте, единиц" + row_1[35:38] = [ + "Средняя сменность работы", + "Количество рабочих мест без оборудования", + "Примечание", + ] + row_2[10], row_2[15], row_2[20], row_2[25], row_2[30] = ( + "До 5 лет", + "От 5 до 10 лет", + "От 10 до 15 лет", + "От 15 до 20 лет", + "Свыше 20 лет", + ) + row_3[6:10] = [ + total_header, + "Введенное в эксплуатацию в отчетном году", + "Выведенное из эксплуатации в отчетном году", + "Импортное оборудование", + ] + for start in (10, 15, 20, 25, 30): + row_3[start : start + 5] = [ + "Всего", + "Импортного оборудования", + "Коэффициент использования оборудования в производстве", + "Доля фактического времени работы оборудования, использованного для производства ПВН, %", + "Средневзвешенный коэффициент загрузки", + ] + for row in (row_1, row_2, row_3, row_4): + sheet.append(row) + + ages = [age_under_5, 25, 20, 20, 15] + imported = [4, 5, 4, 4, 3] + values = [ + "Тестовая организация Ф-6", + "90000001", + "1267700000017", + "6234567890", + "101", + "Основное технологическое оборудование", + total_equipment, + 12, + 7, + sum(imported), + ] + for total, imported_total in zip(ages, imported, strict=True): + values.extend([total, imported_total, 75, 20, 0.8]) + values.extend([1.5, 4, "Тестовые согласованные данные"]) + sheet.append(values) + + file = BytesIO() + workbook.save(file) + file.seek(0) + return file + def test_get_column_mappings_returns_mappings(self): """Test get_column_mappings returns correct mappings.""" parser = FormF6Parser(report_year=2026, report_quarter=1) @@ -59,6 +140,41 @@ class FormF6ParserTest(TestCase): field_names = [m.field_name for m in mappings] self.assertIn("row_code", field_names) self.assertIn("total_equipment", field_names) + self.assertIn("commissioned_equipment", field_names) + self.assertIn("age_over_20_imported", field_names) + + def test_parse_current_template_maps_all_age_buckets(self): + OrganizationFactory.create(inn="6234567890") + parser = FormF6Parser(report_year=2026, report_quarter=1) + + result = parser.parse(self._current_template_file()) + + self.assertEqual(result.loaded_count, 1) + record = FormF6Record.objects.get() + self.assertEqual(record.total_equipment, 100) + self.assertEqual(record.commissioned_equipment, 12) + self.assertEqual(record.decommissioned_equipment, 7) + self.assertEqual(record.age_under_5, 20) + self.assertEqual(record.age_over_20, 15) + self.assertEqual(record.age_over_20_imported, 3) + + def test_rejects_non_integral_equipment_counts_before_creating_records(self): + OrganizationFactory.create(inn="6234567890") + parser = FormF6Parser(report_year=2026, report_quarter=1) + + with self.assertRaises(ExcelValidationError): + parser.parse(self._current_template_file(age_under_5=20.5)) + + self.assertFalse(FormF6Record.objects.exists()) + + def test_rejects_incompatible_template_before_creating_records(self): + OrganizationFactory.create(inn="6234567890") + parser = FormF6Parser(report_year=2026, report_quarter=1) + + with self.assertRaises(ExcelValidationError): + parser.parse(self._current_template_file(total_header="Импортное")) + + self.assertFalse(FormF6Record.objects.exists()) def test_create_record_uses_existing_organization(self): """Report imports reuse existing organizations when available.""" diff --git a/tests/apps/organization/test_analytics_api.py b/tests/apps/organization/test_analytics_api.py index 8f3ada8..07ef618 100644 --- a/tests/apps/organization/test_analytics_api.py +++ b/tests/apps/organization/test_analytics_api.py @@ -41,6 +41,8 @@ class OrganizationAnalyticsApiTest(APITestCase): organization=self.organization, report_year=2026, report_quarter=1, + report_month=6, + avg_employees=1050, avg_payroll_employees=995, payroll_fund=Decimal("1000000.00"), military_output_actual=Decimal("11000000.00"), @@ -56,6 +58,8 @@ class OrganizationAnalyticsApiTest(APITestCase): organization=self.organization, report_year=2025, report_quarter=1, + report_month=12, + avg_employees=1020, avg_payroll_employees=970, payroll_fund=Decimal("900000.00"), military_output_actual=Decimal("9000000.00"), @@ -88,11 +92,19 @@ class OrganizationAnalyticsApiTest(APITestCase): FormF3RecordFactory.create( organization=self.organization, report_year=2026, - avg_employees=1050, - production_workers=620, - engineering_workers=210, - administrative_workers=220, - workers_needed=35, + avg_employees=None, + production_workers=None, + engineering_workers=None, + administrative_workers=None, + workers_needed=None, + average_employee_age=Decimal("42.00"), + employees_under_20=4, + employees_20_29=18, + employees_30_39=28, + employees_40_49=26, + employees_50_59=17, + employees_over_60=7, + machine_tools_and_equipment=46, total_equipment=187, domestic_equipment=91, imported_equipment=96, @@ -109,14 +121,16 @@ class OrganizationAnalyticsApiTest(APITestCase): FormF3RecordFactory.create( organization=self.organization, report_year=2025, - avg_employees=1020, + avg_employees=None, + average_employee_age=Decimal("41.00"), ) FormF4RecordFactory.create( organization=self.organization, report_year=2026, - revenue_rsbu=Decimal("1100000000.00"), - net_profit_rsbu=Decimal("320000000.00"), - ebitda_rsbu=Decimal("480000000.00"), + report_half_year=2, + revenue_rsbu=Decimal("477807.00"), + net_profit_rsbu=Decimal("442414.00"), + ebitda_rsbu=Decimal("138564.00"), gross_profit_rsbu=Decimal("520000000.00"), operating_profit_rsbu=Decimal("300000000.00"), net_debt_rsbu=Decimal("200000000.00"), @@ -131,9 +145,10 @@ class OrganizationAnalyticsApiTest(APITestCase): FormF4RecordFactory.create( organization=self.organization, report_year=2025, - revenue_rsbu=Decimal("980000000.00"), - net_profit_rsbu=Decimal("250000000.00"), - ebitda_rsbu=Decimal("410000000.00"), + report_half_year=2, + revenue_rsbu=Decimal("450000.00"), + net_profit_rsbu=Decimal("400000.00"), + ebitda_rsbu=Decimal("120000.00"), ros=Decimal("21.00"), roa=Decimal("10.50"), roe=Decimal("14.10"), @@ -152,14 +167,36 @@ class OrganizationAnalyticsApiTest(APITestCase): organization=self.organization, report_year=2026, category="Станочное оборудование", - total_equipment=54, - domestic_equipment=31, - imported_equipment=23, + total_equipment=187, + domestic_equipment=None, + imported_equipment=96, + commissioned_equipment=12, + decommissioned_equipment=7, age_under_5=70, + age_under_5_imported=35, + age_under_5_utilization_rate=Decimal("92.00"), + age_under_5_lease_share_itn_percent=Decimal("20.00"), + age_under_5_weighted_load_factor=Decimal("0.78"), age_5_10=41, + age_5_10_imported=21, + age_5_10_utilization_rate=Decimal("92.00"), + age_5_10_lease_share_itn_percent=Decimal("20.00"), + age_5_10_weighted_load_factor=Decimal("0.74"), age_10_15=33, + age_10_15_imported=17, + age_10_15_utilization_rate=Decimal("92.00"), + age_10_15_lease_share_itn_percent=Decimal("20.00"), + age_10_15_weighted_load_factor=Decimal("0.69"), age_15_20=22, + age_15_20_imported=12, + age_15_20_utilization_rate=Decimal("92.00"), + age_15_20_lease_share_itn_percent=Decimal("20.00"), + age_15_20_weighted_load_factor=Decimal("0.63"), age_over_20=21, + age_over_20_imported=11, + age_over_20_utilization_rate=Decimal("92.00"), + age_over_20_lease_share_itn_percent=Decimal("20.00"), + age_over_20_weighted_load_factor=Decimal("0.57"), physical_wear_percent=Decimal("28.40"), utilization_rate=Decimal("92.00"), avg_shift_work=Decimal("1.80"), @@ -231,14 +268,24 @@ class OrganizationAnalyticsApiTest(APITestCase): self.assertEqual(response.data["organization_id"], str(self.organization.id)) self.assertEqual(response.data["group"], "efficiency") self.assertEqual(response.data["periods"], [2025, 2026]) + self.assertEqual( + response.data["report_periods"], + [ + {"year": 2025, "report_half_year": 2}, + {"year": 2026, "report_half_year": 2}, + ], + ) self.assertEqual( response.data["kpis"].keys(), {"revenue", "ebitda", "net_profit", "revenue_per_employee"}, ) - self.assertGreater(response.data["kpis"]["revenue_per_employee"]["value"], 0) + self.assertEqual(response.data["kpis"]["revenue"]["value"], 477807) + self.assertEqual(response.data["kpis"]["net_profit"]["value"], 442414) + self.assertEqual(response.data["kpis"]["ebitda"]["value"], 138564) + self.assertEqual(response.data["kpis"]["revenue_per_employee"]["value"], 455) self.assertEqual( response.data["kpis"]["revenue_per_employee"]["unit"], - "rub_per_employee", + "rub_thousands_per_employee", ) for kpi in response.data["kpis"].values(): self.assertIn("previous_value", kpi) @@ -251,7 +298,7 @@ class OrganizationAnalyticsApiTest(APITestCase): self.assertIn("unit", series) self.assertIn("points", series) expected_unit = ( - "rub_per_employee" + "rub_thousands_per_employee" if series["metric"] == "revenue_per_employee" else "rub_thousands" ) @@ -275,6 +322,61 @@ class OrganizationAnalyticsApiTest(APITestCase): ) ) + def test_economics_calculates_roa_and_roe_from_f4_and_latest_f2(self): + FormF2RecordFactory.create( + organization=self.organization, + report_year=2025, + report_quarter=4, + total_assets=Decimal("800000.00"), + total_equity=Decimal("400000.00"), + ) + FormF2RecordFactory.create( + organization=self.organization, + report_year=2026, + report_quarter=4, + total_assets=Decimal("884828.00"), + total_equity=Decimal("442414.00"), + ) + + response = self.client.get( + f"/api/v1/organizations/{self.organization.id}/analytics/economics/" + "?group=efficiency&from_year=2025&to_year=2026" + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + ratios_by_period = {row["period"]: row for row in response.data["ratios"]} + self.assertEqual(ratios_by_period[2025]["roa"], 50.0) + self.assertEqual(ratios_by_period[2025]["roe"], 100.0) + self.assertEqual(ratios_by_period[2026]["roa"], 50.0) + self.assertEqual(ratios_by_period[2026]["roe"], 100.0) + self.assertEqual( + response.data["ratios_report_period"], + { + "financial_form": "F-2", + "report_year": 2026, + "report_quarter": 4, + }, + ) + + def test_economics_returns_null_roa_and_roe_without_f2_denominators(self): + organization = OrganizationFactory.create() + FormF4RecordFactory.create( + organization=organization, + report_year=2026, + report_half_year=2, + net_profit_rsbu=Decimal("1000.00"), + ) + + response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/economics/" + "?group=efficiency&from_year=2026&to_year=2026" + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIsNone(response.data["ratios"][0]["roa"]) + self.assertIsNone(response.data["ratios"][0]["roe"]) + self.assertIsNone(response.data["ratios_report_period"]) + def test_economics_respects_requested_range_when_it_is_empty(self): response = self.client.get( f"/api/v1/organizations/{self.organization.id}/analytics/economics/" @@ -283,10 +385,43 @@ class OrganizationAnalyticsApiTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.data["periods"], []) + self.assertEqual(response.data["report_periods"], []) self.assertEqual(response.data["kpis"], {}) self.assertFalse(response.data["data_available"]) self.assertEqual(response.data["message"], "Данные отсутствуют") + def test_economics_normalizes_f2_rubles_to_thousands_without_f4(self): + organization = OrganizationFactory.create() + FormF1RecordFactory.create( + organization=organization, + report_year=2026, + report_month=6, + avg_employees=100, + ) + FormF2RecordFactory.create( + organization=organization, + report_year=2026, + revenue=Decimal("1200000.00"), + net_profit=Decimal("300000.00"), + ebitda=Decimal("480000.00"), + ) + + response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/economics/" + "?group=efficiency&from_year=2026&to_year=2026" + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["kpis"]["revenue"]["value"], 1200) + self.assertEqual( + response.data["kpis"]["revenue_per_employee"]["value"], + 12, + ) + self.assertEqual( + response.data["kpis"]["revenue"]["unit"], + "rub_thousands", + ) + def test_personnel_contract(self): personnel_response = self.client.get( f"/api/v1/organizations/{self.organization.id}/analytics/personnel/" @@ -297,6 +432,7 @@ class OrganizationAnalyticsApiTest(APITestCase): personnel_response.data["organization_id"], str(self.organization.id) ) self.assertEqual(personnel_response.data["report_year"], 2026) + self.assertEqual(personnel_response.data["report_month"], 6) self.assertEqual( personnel_response.data["headcount"]["average_employees"], 1050, @@ -306,6 +442,7 @@ class OrganizationAnalyticsApiTest(APITestCase): set(personnel_response.data["history"][0]), { "year", + "report_month", "average_employees", "avg_payroll_employees", "average_age", @@ -313,15 +450,22 @@ class OrganizationAnalyticsApiTest(APITestCase): }, ) self.assertEqual(len(personnel_response.data["age_distribution"]), 3) - self.assertIn("average_age", personnel_response.data) + self.assertEqual(personnel_response.data["average_age"], 42.0) self.assertEqual( personnel_response.data["headcount"]["avg_payroll_employees"], 995 ) - self.assertEqual(personnel_response.data["headcount"]["payroll_fund"], 1000000) + self.assertEqual( + personnel_response.data["headcount"]["payroll_fund"], 1000000000 + ) + self.assertIsNone(personnel_response.data["headcount"]["production_workers"]) self.assertEqual( personnel_response.data["age_distribution"][0]["age_group"], "under_30", ) + self.assertEqual( + personnel_response.data["age_distribution"][0]["employees_count"], + 22, + ) self.assertIn("employees_count", personnel_response.data["age_distribution"][0]) def test_yearly_analytics_use_latest_available_year_when_requested_year_is_empty( @@ -374,15 +518,19 @@ class OrganizationAnalyticsApiTest(APITestCase): }, ) self.assertEqual(response.data["summary"]["total_equipment"], 187) + self.assertEqual(response.data["summary"]["machine_tools_and_equipment"], 46) + self.assertIsNone(response.data["summary"]["domestic_equipment"]) + self.assertEqual(response.data["summary"]["imported_equipment"], 96) self.assertEqual(response.data["summary"]["physical_wear_percent"], 32.0) - self.assertEqual(response.data["summary"]["weighted_wear_percent"], 32.0) + self.assertIsNone(response.data["summary"]["weighted_wear_percent"]) self.assertEqual(response.data["summary"]["utilization_rate"], 0.92) - self.assertEqual(response.data["summary"]["commissioned_equipment"], 1) - self.assertEqual(response.data["summary"]["decommissioned_equipment"], 1) + self.assertEqual(response.data["summary"]["commissioned_equipment"], 12) + self.assertEqual(response.data["summary"]["decommissioned_equipment"], 7) self.assertEqual( response.data["age_distribution"][0]["bucket"], "under_5_years" ) self.assertEqual(len(response.data["age_distribution"]), 5) + self.assertEqual(response.data["age_distribution"][0]["imported_equipment"], 35) self.assertEqual( set(response.data["age_distribution"][0]), { @@ -392,21 +540,26 @@ class OrganizationAnalyticsApiTest(APITestCase): "utilization_rate", "lease_share_itn_percent", "weighted_wear_percent", + "weighted_load_factor", }, ) + self.assertEqual( + response.data["age_distribution"][0]["weighted_load_factor"], + 0.78, + ) self.assertGreaterEqual(len(response.data["categories"]), 1) self.assertEqual(len(response.data["dynamics_series"]), 3) self.assertEqual( response.data["categories"][0], { "category": "Станочное оборудование", - "total_equipment": 54, - "domestic_equipment": 31, - "imported_equipment": 23, - "physical_wear_percent": 28.4, - "weighted_wear_percent": 28.4, + "total_equipment": 187, + "domestic_equipment": None, + "imported_equipment": 96, + "physical_wear_percent": None, + "weighted_wear_percent": None, "utilization_rate": 0.92, - "lease_share_itn_percent": None, + "lease_share_itn_percent": 20.0, }, ) @@ -422,6 +575,7 @@ class OrganizationAnalyticsApiTest(APITestCase): self.assertEqual(products_response.data["report_year"], 2026) self.assertEqual(products_response.data["frequency"], "quarterly") self.assertEqual(products_response.data["price_mode"], "actual") + self.assertEqual(products_response.data["summary_period"], "2026-Q1") self.assertEqual( products_response.data["summary"]["military_output_amount"], 11000000 ) @@ -479,6 +633,7 @@ class OrganizationAnalyticsApiTest(APITestCase): ) self.assertEqual(semiannual_response.status_code, status.HTTP_200_OK) self.assertEqual(semiannual_response.data["frequency"], "semiannual") + self.assertEqual(semiannual_response.data["summary_period"], "2026-H1") self.assertEqual(len(semiannual_response.data["production_series"]), 1) self.assertEqual( semiannual_response.data["production_series"][0]["period"], "2026-H1" @@ -494,6 +649,7 @@ class OrganizationAnalyticsApiTest(APITestCase): ) self.assertEqual(monthly_response.status_code, status.HTTP_200_OK) self.assertEqual(monthly_response.data["frequency"], "monthly") + self.assertEqual(monthly_response.data["summary_period"], "2026-06") self.assertEqual(len(monthly_response.data["production_series"]), 6) self.assertEqual( monthly_response.data["production_series"][0]["period"], "2026-01" @@ -503,6 +659,104 @@ class OrganizationAnalyticsApiTest(APITestCase): 3666666, ) + def test_products_aggregates_real_monthly_f1_records_chronologically(self): + organization = OrganizationFactory.create() + for month in (7, 4, 5, 2, 1, 3, 6): + amount = Decimal(month) * Decimal("1000000.00") + FormF1RecordFactory.create( + organization=organization, + report_year=2026, + report_month=month, + report_quarter=None, + military_output_actual=amount, + civilian_output_actual=amount, + hightech_output_actual=amount, + rd_volume_actual=amount, + military_domestic_actual=amount, + military_export_actual=amount, + civilian_domestic_actual=amount, + civilian_export_actual=amount, + ) + + monthly_response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/products/" + "?frequency=monthly&price_mode=actual&report_year=2026" + ) + quarterly_response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/products/" + "?frequency=quarterly&price_mode=actual&report_year=2026" + ) + semiannual_response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/products/" + "?frequency=semiannual&price_mode=actual&report_year=2026" + ) + annual_response = self.client.get( + f"/api/v1/organizations/{organization.id}/analytics/products/" + "?frequency=annual&price_mode=actual&report_year=2026" + ) + + for response in ( + monthly_response, + quarterly_response, + semiannual_response, + annual_response, + ): + self.assertEqual(response.status_code, status.HTTP_200_OK) + + self.assertEqual( + [row["period"] for row in monthly_response.data["production_series"]], + [ + "2026-01", + "2026-02", + "2026-03", + "2026-04", + "2026-05", + "2026-06", + "2026-07", + ], + ) + self.assertEqual(monthly_response.data["summary_period"], "2026-07") + self.assertEqual( + monthly_response.data["summary"]["military_output_amount"], + 7000000, + ) + + self.assertEqual( + [row["period"] for row in quarterly_response.data["production_series"]], + ["2026-Q1", "2026-Q2", "2026-Q3"], + ) + self.assertEqual( + [ + row["military_output_amount"] + for row in quarterly_response.data["production_series"] + ], + [6000000, 15000000, 7000000], + ) + self.assertEqual(quarterly_response.data["summary_period"], "2026-Q3") + + self.assertEqual( + [row["period"] for row in semiannual_response.data["production_series"]], + ["2026-H1", "2026-H2"], + ) + self.assertEqual( + [ + row["military_output_amount"] + for row in semiannual_response.data["production_series"] + ], + [21000000, 7000000], + ) + self.assertEqual(semiannual_response.data["summary_period"], "2026-H2") + + self.assertEqual( + [row["period"] for row in annual_response.data["production_series"]], + ["2026"], + ) + self.assertEqual( + annual_response.data["production_series"][0]["military_output_amount"], + 28000000, + ) + self.assertEqual(annual_response.data["summary_period"], "2026") + def test_forecast_contract(self): response = self.client.get( f"/api/v1/organizations/{self.organization.id}/analytics/forecast/" diff --git a/tests/apps/user/test_views.py b/tests/apps/user/test_views.py index 153e05e..c20656d 100644 --- a/tests/apps/user/test_views.py +++ b/tests/apps/user/test_views.py @@ -152,7 +152,7 @@ class CurrentUserViewTest(APITestCase): response = self.client.get(self.current_user_url) self.assertEqual(response.status_code, status.HTTP_200_OK) - self.assertEqual(response.data["role"], "staff") + self.assertEqual(response.data["role"], "admin") self.assertEqual(response.data["capabilities"]["can_access_admin_page"], True) def test_get_current_user_unauthenticated(self): @@ -224,6 +224,10 @@ class AdminUsersManagementViewTest(APITestCase): self.assertEqual(job_user_row["first_name"], "Иван") self.assertEqual(job_user_row["middle_name"], "Сергеевич") self.assertEqual(job_user_row["last_name"], "Петров") + self.assertEqual(job_user_row["profile"]["middle_name"], "Сергеевич") + self.assertEqual(job_user_row["role"], "user") + self.assertEqual(job_user_row["role_label"], "Пользователь") + self.assertFalse(job_user_row["capabilities"]["can_access_admin_page"]) self.assertEqual(job_user_row["progress_message"], "Готово") self.assertEqual(job_user_row["result"], {"processed": 10}) self.assertIsNone(job_user_row["error"]) @@ -237,6 +241,29 @@ class AdminUsersManagementViewTest(APITestCase): self.assertIsNone(regular_user_row["error"]) self.assertIsNone(regular_user_row["is_successful"]) + def test_admin_can_search_order_and_paginate_users(self): + first = UserFactory.create_user(username="alpha-user") + second = UserFactory.create_user(username="beta-user") + first.profile.first_name = "Яков" + first.profile.save(update_fields=["first_name"]) + second.profile.first_name = "Алексей" + second.profile.save(update_fields=["first_name"]) + + search_response = self.client.get(self.url, {"search": "beta-user"}) + ordering_response = self.client.get( + self.url, + {"ordering": "first_name", "page": 1, "page_size": 100}, + ) + + self.assertEqual(search_response.status_code, status.HTTP_200_OK) + self.assertEqual( + [item["id"] for item in search_response.data["results"]], + [second.id], + ) + ordered_ids = [item["id"] for item in ordering_response.data["results"]] + self.assertLess(ordered_ids.index(second.id), ordered_ids.index(first.id)) + self.assertEqual(ordering_response.data["count"], User.objects.count()) + def test_non_admin_cannot_access_endpoint(self): self.client.force_authenticate(self.regular_user) response = self.client.get(self.url) @@ -244,6 +271,167 @@ class AdminUsersManagementViewTest(APITestCase): self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) +class AdminUserCrudViewTest(APITestCase): + """Tests for admin create, detail, update and activation endpoints.""" + + def setUp(self): + self.admin_user = UserFactory.create_user(is_staff=True) + self.regular_user = UserFactory.create_user() + self.client.force_authenticate(self.admin_user) + self.list_url = reverse("api_v1:user:admin_users") + + def test_admin_can_create_user_with_role_and_profile(self): + password = fake.password(length=12, special_chars=False) + payload = { + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "phone": f"+7{fake.numerify('##########')}", + "password": password, + "role": "admin", + "is_verified": True, + "first_name": "Пётр", + "middle_name": "Петрович", + "last_name": "Петров", + } + + response = self.client.post(self.list_url, payload, format="json") + + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + created = User.objects.get(username=payload["username"]) + self.assertTrue(created.check_password(password)) + self.assertTrue(created.is_staff) + self.assertFalse(created.is_superuser) + self.assertTrue(created.is_verified) + self.assertTrue(created.groups.filter(name="admin").exists()) + self.assertEqual(created.profile.first_name, "Пётр") + self.assertEqual(created.profile.mid_name, "Петрович") + self.assertEqual(response.data["role"], "admin") + + def test_admin_create_validates_unique_fields_and_password(self): + duplicate_response = self.client.post( + self.list_url, + { + "email": self.regular_user.email, + "username": fake.unique.user_name(), + "password": "valid-pass", + "first_name": "Иван", + "last_name": "Иванов", + }, + format="json", + ) + password_response = self.client.post( + self.list_url, + { + "email": fake.unique.email(), + "username": fake.unique.user_name(), + "password": "short", + "first_name": "Иван", + "last_name": "Иванов", + }, + format="json", + ) + + self.assertEqual(duplicate_response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(password_response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_admin_can_get_and_update_user(self): + detail_url = reverse( + "api_v1:user:admin-user-detail", + args=[self.regular_user.id], + ) + new_password = fake.password(length=12, special_chars=False) + + detail_response = self.client.get(detail_url) + update_response = self.client.patch( + detail_url, + { + "role": "admin", + "password": new_password, + "is_verified": True, + "first_name": "Иван", + "middle_name": "Иванович", + "last_name": "Иванов", + }, + format="json", + ) + + self.assertEqual(detail_response.status_code, status.HTTP_200_OK) + self.assertIn("profile", detail_response.data) + self.assertIn("progress_message", detail_response.data) + self.assertEqual(update_response.status_code, status.HTTP_200_OK) + self.regular_user.refresh_from_db() + self.assertTrue(self.regular_user.is_staff) + self.assertTrue(self.regular_user.is_verified) + self.assertTrue(self.regular_user.check_password(new_password)) + self.assertEqual(self.regular_user.profile.first_name, "Иван") + self.assertEqual(self.regular_user.profile.mid_name, "Иванович") + self.assertEqual(update_response.data["role"], "admin") + + def test_admin_can_deactivate_and_activate_user(self): + deactivate_url = reverse( + "api_v1:user:admin-user-deactivate", + args=[self.regular_user.id], + ) + activate_url = reverse( + "api_v1:user:admin-user-activate", + args=[self.regular_user.id], + ) + + deactivate_response = self.client.post(deactivate_url, {}, format="json") + self.regular_user.refresh_from_db() + self.assertEqual(deactivate_response.status_code, status.HTTP_200_OK) + self.assertFalse(self.regular_user.is_active) + + activate_response = self.client.post(activate_url, {}, format="json") + self.regular_user.refresh_from_db() + self.assertEqual(activate_response.status_code, status.HTTP_200_OK) + self.assertTrue(self.regular_user.is_active) + + def test_admin_cannot_deactivate_or_demote_self(self): + detail_url = reverse( + "api_v1:user:admin-user-detail", + args=[self.admin_user.id], + ) + deactivate_url = reverse( + "api_v1:user:admin-user-deactivate", + args=[self.admin_user.id], + ) + + demote_response = self.client.patch( + detail_url, + {"role": "user"}, + format="json", + ) + patch_inactive_response = self.client.patch( + detail_url, + {"is_active": False}, + format="json", + ) + deactivate_response = self.client.post(deactivate_url, {}, format="json") + + self.assertEqual(demote_response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + patch_inactive_response.status_code, + status.HTTP_400_BAD_REQUEST, + ) + self.assertEqual(deactivate_response.status_code, status.HTTP_400_BAD_REQUEST) + + def test_regular_user_cannot_manage_users(self): + self.client.force_authenticate(self.regular_user) + detail_url = reverse( + "api_v1:user:admin-user-detail", + args=[self.admin_user.id], + ) + + response = self.client.patch( + detail_url, + {"is_active": False}, + format="json", + ) + + self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) + + class UserUpdateViewTest(APITestCase): """Tests for UserUpdateView"""