From d112d1715fe5d866589c6015bde9ed74ae4125d1 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Tue, 2 Jun 2026 00:28:09 +0200 Subject: [PATCH] fix: create organizations from form uploads --- src/apps/core/excel.py | 2 +- src/apps/form_1/api.py | 54 ++++++++ src/apps/form_1/services.py | 8 +- src/apps/form_2/services.py | 8 +- src/apps/form_3/services.py | 8 +- src/apps/form_4/services.py | 8 +- src/apps/form_5/services.py | 8 +- src/apps/form_6/services.py | 8 +- src/apps/organization/services.py | 61 ++++++++- tests/apps/form_1/test_api.py | 82 +++++++++++ tests/apps/form_1/test_services.py | 2 +- tests/apps/form_2/test_services.py | 2 +- tests/apps/form_3/test_services.py | 2 +- tests/apps/form_4/test_services.py | 2 +- tests/apps/form_5/test_services.py | 2 +- tests/apps/form_6/test_services.py | 2 +- .../test_parser_organization_creation.py | 127 ++++++++++++++++++ 17 files changed, 372 insertions(+), 14 deletions(-) create mode 100644 tests/apps/form_1/test_api.py create mode 100644 tests/apps/forms/test_parser_organization_creation.py diff --git a/src/apps/core/excel.py b/src/apps/core/excel.py index 940377d..4645214 100644 --- a/src/apps/core/excel.py +++ b/src/apps/core/excel.py @@ -253,7 +253,7 @@ class BaseExcelParser(ABC, Generic[T]): ] def create_record(self, row_data: RowData) -> FormF1Record: - org = OrganizationService.get_required_by_inn(...) + org = OrganizationService.get_or_create_from_form_identifiers(...) return FormF1Record.objects.create(organization=org, ...) """ diff --git a/src/apps/form_1/api.py b/src/apps/form_1/api.py index 194039a..a4cd4b5 100644 --- a/src/apps/form_1/api.py +++ b/src/apps/form_1/api.py @@ -30,6 +30,7 @@ from django_filters import rest_framework as filters from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.decorators import action +from rest_framework.filters import OrderingFilter, SearchFilter from rest_framework.parsers import MultiPartParser from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request @@ -41,6 +42,48 @@ logger = logging.getLogger(__name__) # Порог для фоновой обработки (количество строк) BACKGROUND_THRESHOLD = 100 +_PERIOD_ORDERING_FIELDS = {"report_year", "report_quarter"} +_ALL_PERIOD_ORDERING_FIELDS = _PERIOD_ORDERING_FIELDS | {"report_month"} + + +def _field_name(ordering_item: str) -> str: + return ordering_item.lstrip("-") + + +def _period_ordering_direction(ordering: list[str]) -> str: + for ordering_item in ordering: + if _field_name(ordering_item) in _ALL_PERIOD_ORDERING_FIELDS: + return "-" if ordering_item.startswith("-") else "" + return "-" + + +def _with_form_f1_period_tiebreakers(ordering: list[str]) -> list[str]: + ordered_fields = {_field_name(ordering_item) for ordering_item in ordering} + if not (ordered_fields & _PERIOD_ORDERING_FIELDS): + return ordering + + resolved_ordering = list(ordering) + direction = _period_ordering_direction(resolved_ordering) + + if "report_month" not in ordered_fields: + resolved_ordering.append(f"{direction}report_month") + + if "created_at" not in ordered_fields: + resolved_ordering.append("-created_at") + + return resolved_ordering + + +class FormF1OrderingFilter(OrderingFilter): + """Adds monthly period tiebreakers for F-1 ordering requests.""" + + def get_ordering(self, request, queryset, view): + ordering = super().get_ordering(request, queryset, view) + if not ordering: + return ordering + + return _with_form_f1_period_tiebreakers(list(ordering)) + class FormF1Filter(filters.FilterSet): """Фильтры для записей формы Ф-1.""" @@ -168,6 +211,11 @@ class FormF1RecordViewSet(ReadOnlyViewSet[FormF1Record]): ) serializer_class = FormF1RecordSerializer permission_classes = [IsAuthenticated] + filter_backends = [ + filters.DjangoFilterBackend, + SearchFilter, + FormF1OrderingFilter, + ] filterset_class = FormF1Filter search_fields = ["organization__name", "organization__inn"] ordering_fields = [ @@ -189,6 +237,12 @@ class FormF1RecordViewSet(ReadOnlyViewSet[FormF1Record]): return self.serializer_classes[self.action] return super().get_serializer_class() + def retrieve(self, request: Request, *args, **kwargs) -> Response: + """Return plain detail payload matching the generated F-1 API contract.""" + instance = self.get_object() + serializer = self.get_serializer(instance) + return Response(serializer.data) + @swagger_auto_schema( tags=["Форма Ф-1"], operation_summary="Список загрузок", diff --git a/src/apps/form_1/services.py b/src/apps/form_1/services.py index 6e46672..59f7bb6 100644 --- a/src/apps/form_1/services.py +++ b/src/apps/form_1/services.py @@ -275,7 +275,13 @@ class FormF1Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF1Record]): """Создать запись формы Ф-1.""" row_data = self._normalize_row_data(row_data) batch_id = batch_id or getattr(self, "load_batch", self.get_next_batch_id()) - org = OrganizationService.get_required_by_inn(row_data.inn) + org = OrganizationService.get_or_create_from_form_identifiers( + name=row_data.organization_name, + inn=row_data.inn, + ogrn=row_data.ogrn, + kpp=row_data.kpp, + okpo=row_data.okpo, + ) extra_period_fields = {} report_quarter = self.report_quarter if self.report_month is not None: diff --git a/src/apps/form_2/services.py b/src/apps/form_2/services.py index 3f78649..fdf18c6 100644 --- a/src/apps/form_2/services.py +++ b/src/apps/form_2/services.py @@ -366,7 +366,13 @@ class FormF2Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF2Record]): """Создать запись формы Ф-2.""" row_data = self._normalize_row_data(row_data) batch_id = batch_id or getattr(self, "load_batch", self.get_next_batch_id()) - org = OrganizationService.get_required_by_inn(row_data.inn) + org = OrganizationService.get_or_create_from_form_identifiers( + name=row_data.organization_name, + inn=row_data.inn, + ogrn=row_data.ogrn, + kpp=row_data.kpp, + okpo=row_data.okpo, + ) record = FormF2Service.create_versioned_record( organization=org, diff --git a/src/apps/form_3/services.py b/src/apps/form_3/services.py index d79d1ea..386266b 100644 --- a/src/apps/form_3/services.py +++ b/src/apps/form_3/services.py @@ -185,7 +185,13 @@ class FormF3Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF3Record]): """Создать запись формы Ф-3.""" row_data = self._normalize_row_data(row_data) batch_id = batch_id or getattr(self, "load_batch", self.get_next_batch_id()) - org = OrganizationService.get_required_by_inn(row_data.inn) + org = OrganizationService.get_or_create_from_form_identifiers( + name=row_data.organization_name, + inn=row_data.inn, + ogrn=row_data.ogrn, + kpp=row_data.kpp, + okpo=row_data.okpo, + ) record = FormF3Service.create_versioned_record( organization=org, diff --git a/src/apps/form_4/services.py b/src/apps/form_4/services.py index 734e765..42c5a7f 100644 --- a/src/apps/form_4/services.py +++ b/src/apps/form_4/services.py @@ -166,7 +166,13 @@ class FormF4Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF4Record]): ) -> FormF4Record: row_data = self._normalize_row_data(row_data) batch_id = batch_id or getattr(self, "load_batch", self.get_next_batch_id()) - org = OrganizationService.get_required_by_inn(row_data.inn) + org = OrganizationService.get_or_create_from_form_identifiers( + name=row_data.organization_name, + inn=row_data.inn, + ogrn=row_data.ogrn, + kpp=row_data.kpp, + okpo=row_data.okpo, + ) report_half_year = self.report_half_year if report_half_year is None and self.report_quarter in {1, 2}: report_half_year = self.report_quarter diff --git a/src/apps/form_5/services.py b/src/apps/form_5/services.py index 1c1a3b2..ef801f2 100644 --- a/src/apps/form_5/services.py +++ b/src/apps/form_5/services.py @@ -148,7 +148,13 @@ class FormF5Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF5Record]): ) -> FormF5Record: row_data = self._normalize_row_data(row_data) batch_id = batch_id or getattr(self, "load_batch", self.get_next_batch_id()) - org = OrganizationService.get_required_by_inn(row_data.inn) + org = OrganizationService.get_or_create_from_form_identifiers( + name=row_data.organization_name, + inn=row_data.inn, + ogrn=row_data.ogrn, + kpp=row_data.kpp, + okpo=row_data.okpo, + ) return FormF5Service.create_versioned_record( organization=org, load_batch=batch_id, diff --git a/src/apps/form_6/services.py b/src/apps/form_6/services.py index 6bdea80..27d9e91 100644 --- a/src/apps/form_6/services.py +++ b/src/apps/form_6/services.py @@ -135,7 +135,13 @@ class FormF6Parser(ReportingPeriodParserMixin, BaseExcelParser[FormF6Record]): ) -> FormF6Record: row_data = self._normalize_row_data(row_data) batch_id = batch_id or getattr(self, "load_batch", self.get_next_batch_id()) - org = OrganizationService.get_required_by_inn(row_data.inn) + org = OrganizationService.get_or_create_from_form_identifiers( + name=row_data.organization_name, + inn=row_data.inn, + ogrn=row_data.ogrn, + kpp=row_data.kpp, + okpo=row_data.okpo, + ) return FormF6Service.create_versioned_record( organization=org, load_batch=batch_id, diff --git a/src/apps/organization/services.py b/src/apps/organization/services.py index addd5fc..0e8bf54 100644 --- a/src/apps/organization/services.py +++ b/src/apps/organization/services.py @@ -7,7 +7,7 @@ from apps.organization.models import Organization class OrganizationNotFoundError(ValueError): - """Raised when a non-Mostovik import references an unknown organization.""" + """Raised when organization identifiers cannot be resolved.""" class OrganizationService(BaseService[Organization]): @@ -16,6 +16,7 @@ class OrganizationService(BaseService[Organization]): Методы: get_or_create_by_inn: Legacy lookup API. Creation is intentionally disabled. + get_or_create_from_form_identifiers: Создать организацию из строки формы. update_organization: Обновить данные организации search_by_name: Поиск по наименованию """ @@ -53,6 +54,58 @@ class OrganizationService(BaseService[Organization]): ) return organization + @classmethod + def get_or_create_from_form_identifiers( + cls, + *, + name: str | None, + inn: str | None, + ogrn: str | None = None, + kpp: str | None = None, + okpo: str | None = None, + ) -> Organization: + """ + Return an organization for a report row, creating it when missing. + + Form imports are allowed to create the organization reference when the row + contains the standard identifiers. Existing organizations are not + overwritten; the form row only fills identifier fields that are still blank. + """ + normalized_inn = cls._clean_digits(inn) + if not normalized_inn: + raise OrganizationNotFoundError("ИНН организации не указан") + + normalized_name = cls._clean_string(name) + if not normalized_name: + raise OrganizationNotFoundError("Наименование организации не указано") + + organization, created = cls.model.objects.get_or_create( + inn=normalized_inn, + defaults={ + "name": normalized_name, + "ogrn": cls._clean_digits(ogrn), + "kpp": cls._clean_digits(kpp), + "okpo": cls._clean_digits(okpo), + }, + ) + if created: + return organization + + update_fields: list[str] = [] + for field_name, value in ( + ("ogrn", cls._clean_digits(ogrn)), + ("kpp", cls._clean_digits(kpp)), + ("okpo", cls._clean_digits(okpo)), + ): + if value and not getattr(organization, field_name): + setattr(organization, field_name, value) + update_fields.append(field_name) + + if update_fields: + organization.save(update_fields=update_fields + ["updated_at"]) + + return organization + @classmethod def search_by_name(cls, query: str, limit: int = 20): """ @@ -91,3 +144,9 @@ class OrganizationService(BaseService[Organization]): if value is None: return "" return "".join(char for char in str(value).strip() if char.isdigit()) + + @staticmethod + def _clean_string(value: Any) -> str: + if value is None: + return "" + return str(value).strip() diff --git a/tests/apps/form_1/test_api.py b/tests/apps/form_1/test_api.py new file mode 100644 index 0000000..26ab5ca --- /dev/null +++ b/tests/apps/form_1/test_api.py @@ -0,0 +1,82 @@ +"""API tests for form_1 records.""" + +from __future__ import annotations + +from decimal import Decimal + +from django.db import connection +from django.test import override_settings +from django.test.utils import CaptureQueriesContext +from rest_framework import status +from rest_framework.test import APITestCase + +from tests.apps.form_1.factories import FormF1RecordFactory +from tests.apps.organization.factories import OrganizationFactory +from tests.apps.user.factories import UserFactory + + +@override_settings(ROOT_URLCONF="core.urls") +class FormF1RecordApiOrderingTest(APITestCase): + """Ordering behavior for monthly F-1 records.""" + + def setUp(self): + self.user = UserFactory.create_user() + self.client.force_authenticate(self.user) + self.organization = OrganizationFactory.create(inn="1111111111") + + def test_ordering_appends_report_month_tiebreaker_for_monthly_records(self): + FormF1RecordFactory.create( + organization=self.organization, + report_year=2025, + report_month=9, + report_quarter=None, + ) + FormF1RecordFactory.create( + organization=self.organization, + report_year=2025, + report_month=12, + report_quarter=None, + ) + + with CaptureQueriesContext(connection) as queries: + response = self.client.get( + "/api/v1/forms/f1/records/", + { + "organization_inn": self.organization.inn, + "ordering": "-report_year,-report_quarter", + "page_size": 100, + }, + ) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + [record["report_month"] for record in response.data["data"]], + [12, 9], + ) + list_query = next( + query["sql"] + for query in queries.captured_queries + if 'FROM "form_1_formf1record"' in query["sql"] + and "ORDER BY" in query["sql"] + ) + self.assertIn('"form_1_formf1record"."report_month" DESC', list_query) + + def test_read_returns_form_f1_record_payload_without_response_envelope(self): + record = FormF1RecordFactory.create( + organization=self.organization, + report_year=2025, + report_month=12, + report_quarter=None, + avg_employees=Decimal("785.00"), + avg_payroll_employees=Decimal("787.00"), + payroll_fund=Decimal("266614.00"), + ) + + response = self.client.get(f"/api/v1/forms/f1/records/{record.id}/") + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["id"], str(record.id)) + self.assertEqual(response.data["avg_employees"], "785.00") + self.assertEqual(response.data["avg_payroll_employees"], "787.00") + self.assertEqual(response.data["payroll_fund"], "266614.00") + self.assertNotIn("data", response.data) diff --git a/tests/apps/form_1/test_services.py b/tests/apps/form_1/test_services.py index 924ba38..3b3a80b 100644 --- a/tests/apps/form_1/test_services.py +++ b/tests/apps/form_1/test_services.py @@ -92,7 +92,7 @@ class FormF1ParserTest(TestCase): self.assertIn("civilian_output_actual", field_names) def test_create_record_uses_existing_organization(self): - """Report imports must attach rows only to preloaded Mostovik organizations.""" + """Report imports reuse existing organizations when available.""" parser = FormF1Parser(report_year=2026, report_month=9) parser.load_batch = 505 organization = OrganizationFactory.create(inn="1234567890") diff --git a/tests/apps/form_2/test_services.py b/tests/apps/form_2/test_services.py index 8415028..1f336b5 100644 --- a/tests/apps/form_2/test_services.py +++ b/tests/apps/form_2/test_services.py @@ -61,7 +61,7 @@ class FormF2ParserTest(TestCase): self.assertIn("revenue", field_names) def test_create_record_uses_existing_organization(self): - """Report imports must attach rows only to preloaded Mostovik organizations.""" + """Report imports reuse existing organizations when available.""" parser = FormF2Parser(report_year=2026, report_quarter=1) parser.load_batch = 502 organization = OrganizationFactory.create(inn="2234567890") diff --git a/tests/apps/form_3/test_services.py b/tests/apps/form_3/test_services.py index c8650f0..bb1d3fc 100644 --- a/tests/apps/form_3/test_services.py +++ b/tests/apps/form_3/test_services.py @@ -61,7 +61,7 @@ class FormF3ParserTest(TestCase): self.assertIn("total_equipment", field_names) def test_create_record_uses_existing_organization(self): - """Report imports must attach rows only to preloaded Mostovik organizations.""" + """Report imports reuse existing organizations when available.""" parser = FormF3Parser(report_year=2026, report_quarter=1) parser.load_batch = 503 organization = OrganizationFactory.create(inn="3234567890") diff --git a/tests/apps/form_4/test_services.py b/tests/apps/form_4/test_services.py index d5cf841..fafe7ad 100644 --- a/tests/apps/form_4/test_services.py +++ b/tests/apps/form_4/test_services.py @@ -61,7 +61,7 @@ class FormF4ParserTest(TestCase): self.assertIn("net_profit_rsbu", field_names) def test_create_record_uses_existing_organization(self): - """Report imports must attach rows only to preloaded Mostovik organizations.""" + """Report imports reuse existing organizations when available.""" parser = FormF4Parser(report_year=2026, report_half_year=1) parser.load_batch = 504 organization = OrganizationFactory.create(inn="4234567890") diff --git a/tests/apps/form_5/test_services.py b/tests/apps/form_5/test_services.py index d505575..ce526f0 100644 --- a/tests/apps/form_5/test_services.py +++ b/tests/apps/form_5/test_services.py @@ -87,7 +87,7 @@ class FormF5ParserTest(TestCase): self.assertIn("name", field_names) def test_create_record_uses_existing_organization(self): - """Report imports must attach rows only to preloaded Mostovik organizations.""" + """Report imports reuse existing organizations when available.""" parser = FormF5Parser(report_year=2026, report_quarter=1) parser.load_batch = 505 organization = OrganizationFactory.create(inn="5234567890") diff --git a/tests/apps/form_6/test_services.py b/tests/apps/form_6/test_services.py index f195f44..fad00db 100644 --- a/tests/apps/form_6/test_services.py +++ b/tests/apps/form_6/test_services.py @@ -61,7 +61,7 @@ class FormF6ParserTest(TestCase): self.assertIn("total_equipment", field_names) def test_create_record_uses_existing_organization(self): - """Report imports must attach rows only to preloaded Mostovik organizations.""" + """Report imports reuse existing organizations when available.""" parser = FormF6Parser(report_year=2026, report_quarter=1) parser.load_batch = 506 organization = OrganizationFactory.create(inn="6234567890") diff --git a/tests/apps/forms/test_parser_organization_creation.py b/tests/apps/forms/test_parser_organization_creation.py new file mode 100644 index 0000000..94e4021 --- /dev/null +++ b/tests/apps/forms/test_parser_organization_creation.py @@ -0,0 +1,127 @@ +"""Tests for organization creation during form imports.""" + +from apps.form_1.services import FormF1Parser +from apps.form_2.services import FormF2Parser +from apps.form_3.services import FormF3Parser +from apps.form_4.services import FormF4Parser +from apps.form_5.services import FormF5Parser +from apps.form_6.services import FormF6Parser +from apps.organization.models import Organization +from django.test import TestCase + + +class FormParserOrganizationCreationTest(TestCase): + """Tests that report rows may create organizations without exchange preload.""" + + CASES = ( + ( + "f1", + FormF1Parser, + {"report_year": 2026, "report_month": 9}, + 601, + { + "organization_name": "Тестовая организация Ф-1", + "inn": "7100000001", + "ogrn": "1020000000001", + "kpp": "770100001", + "okpo": "11110001", + "military_output_actual": 100.0, + "civilian_output_actual": 200.0, + }, + ), + ( + "f2", + FormF2Parser, + {"report_year": 2026, "report_quarter": 1}, + 602, + { + "organization_name": "Тестовая организация Ф-2", + "inn": "7100000002", + "ogrn": "1020000000002", + "kpp": "770100002", + "okpo": "11110002", + "total_assets": 1000000.0, + "revenue": 500000.0, + }, + ), + ( + "f3", + FormF3Parser, + {"report_year": 2026, "report_quarter": 1}, + 603, + { + "organization_name": "Тестовая организация Ф-3", + "inn": "7100000003", + "ogrn": "1020000000003", + "kpp": "770100003", + "okpo": "11110003", + "avg_employees": 100, + "total_equipment": 50, + }, + ), + ( + "f4", + FormF4Parser, + {"report_year": 2026, "report_half_year": 1}, + 604, + { + "organization_name": "Тестовая организация Ф-4", + "inn": "7100000004", + "ogrn": "1020000000004", + "kpp": "770100004", + "okpo": "11110004", + "revenue_rsbu": 1000000.0, + "net_profit_rsbu": 50000.0, + }, + ), + ( + "f5", + FormF5Parser, + {"report_year": 2026, "report_quarter": 1}, + 605, + { + "organization_name": "Тестовая организация Ф-5", + "inn": "7100000005", + "ogrn": "1020000000005", + "kpp": "770100005", + "okpo": "11110005", + "equipment_id": "EQ-001", + "equipment_name": "Токарный станок", + }, + ), + ( + "f6", + FormF6Parser, + {"report_year": 2026, "report_quarter": 1}, + 606, + { + "organization_name": "Тестовая организация Ф-6", + "inn": "7100000006", + "ogrn": "1020000000006", + "kpp": "770100006", + "okpo": "11110006", + "row_code": "001", + "category": "Металлорежущее", + "total_equipment": 100, + }, + ), + ) + + def test_create_record_creates_missing_organization_from_row_identifiers(self): + for form_name, parser_class, parser_kwargs, batch_id, row_data in self.CASES: + with self.subTest(form=form_name): + parser = parser_class(**parser_kwargs) + parser.load_batch = batch_id + + self.assertFalse( + Organization.objects.filter(inn=row_data["inn"]).exists() + ) + + record = parser.create_record(row_data) + + organization = Organization.objects.get(inn=row_data["inn"]) + self.assertEqual(record.organization_id, organization.id) + self.assertEqual(organization.name, row_data["organization_name"]) + self.assertEqual(organization.ogrn, row_data["ogrn"]) + self.assertEqual(organization.kpp, row_data["kpp"]) + self.assertEqual(organization.okpo, row_data["okpo"])