diff --git a/src/apps/backups/services.py b/src/apps/backups/services.py index 5345111..34c6a51 100644 --- a/src/apps/backups/services.py +++ b/src/apps/backups/services.py @@ -28,7 +28,7 @@ from apps.parsers.models import ( ManufacturerRecord, ProcurementRecord, ) -from apps.registers.models import ( +from registers.models import ( Organization, Register, RegisterUpload, @@ -37,7 +37,7 @@ from apps.registers.models import ( from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.db import IntegrityError, transaction -from django.db.models import Model, Q +from django.db.models import Model from django.utils import timezone @@ -127,8 +127,7 @@ class BackupExportService: @classmethod def _get_active_organization_ids(cls, actual_date: date) -> list[int]: return list( - RegistryMembershipPeriod.objects.filter(started_at__lte=actual_date) - .filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date)) + RegistryMembershipPeriod.objects.all() .values_list("organization_id", flat=True) .distinct() ) @@ -140,16 +139,17 @@ class BackupExportService: actual_date: date, active_org_ids: list[int], ) -> dict: - active_periods = RegistryMembershipPeriod.objects.filter( + active_memberships = RegistryMembershipPeriod.objects.filter( organization_id__in=active_org_ids, - started_at__lte=actual_date, - ).filter(Q(ended_at__isnull=True) | Q(ended_at__gt=actual_date)) + ) register_ids = list( - active_periods.values_list("registry_id", flat=True).distinct() + active_memberships.values_list("registry_id", flat=True).distinct() ) upload_ids = list( - active_periods.values_list("started_by_upload_id", flat=True).distinct() + RegisterUpload.objects.filter(registry_id__in=register_ids) + .values_list("id", flat=True) + .distinct() ) reports_qs = FinancialReport.objects.filter( @@ -165,10 +165,9 @@ class BackupExportService: RegisterUpload: RegisterUpload.objects.filter(id__in=upload_ids).order_by( "id" ), - RegistryMembershipPeriod: active_periods.order_by( + RegistryMembershipPeriod: active_memberships.order_by( "registry_id", "organization_id", - "started_at", ), IndustrialCertificateRecord: IndustrialCertificateRecord.objects.filter( registry_organization_id__in=active_org_ids diff --git a/src/apps/core/admin_dashboard.py b/src/apps/core/admin_dashboard.py index 516a7cb..03cc9d6 100644 --- a/src/apps/core/admin_dashboard.py +++ b/src/apps/core/admin_dashboard.py @@ -16,13 +16,13 @@ from apps.parsers.models import ( Proxy, ) from apps.parsers.source_cards import SourceCardService -from apps.registers.models import ( +from registers.models import ( Organization, Register, RegisterUpload, RegistryMembershipPeriod, ) -from django.db.models import Count, Max, Q +from django.db.models import Count, Max from django.urls import NoReverseMatch, reverse SOURCE_COLORS = ( @@ -138,7 +138,7 @@ def build_admin_dashboard() -> dict[str, Any]: source_cards = _build_source_cards() source_mix = _build_source_mix(source_cards) active_registry_orgs = ( - RegistryMembershipPeriod.objects.filter(ended_at__isnull=True) + RegistryMembershipPeriod.objects .values("organization_id") .distinct() .count() @@ -501,8 +501,7 @@ def _build_registry_rows() -> list[dict[str, Any]]: registries = list( Register.objects.annotate( active_organizations=Count( - "membership_periods__organization", - filter=Q(membership_periods__ended_at__isnull=True), + "membership_periods", distinct=True, ), uploads_count=Count("uploads", distinct=True), diff --git a/src/apps/core/logging.py b/src/apps/core/logging.py index c8c8341..aaf837e 100644 --- a/src/apps/core/logging.py +++ b/src/apps/core/logging.py @@ -24,7 +24,7 @@ class JSONFormatter(logging.Formatter): { "timestamp": "2024-01-15T10:30:45.123456Z", "level": "INFO", - "logger": "apps.user.services", + "logger": "user.services", "message": "User created", "request_id": "abc-123", "user_id": 42, diff --git a/src/apps/core/signals.py b/src/apps/core/signals.py index 5d831d4..13a57e2 100644 --- a/src/apps/core/signals.py +++ b/src/apps/core/signals.py @@ -32,7 +32,7 @@ class SignalDispatcher: class UserConfig(AppConfig): def ready(self): - from apps.user.signals import register_signals + from user.signals import register_signals register_signals(signal_dispatcher) Пример в signals.py приложения: diff --git a/src/apps/exchange/state_corp_services.py b/src/apps/exchange/state_corp_services.py index a380b24..47af715 100644 --- a/src/apps/exchange/state_corp_services.py +++ b/src/apps/exchange/state_corp_services.py @@ -23,7 +23,7 @@ from apps.parsers.models import ( InspectionRecord, ProcurementRecord, ) -from apps.registers.models import Organization +from registers.models import Organization from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.utils import timezone diff --git a/src/apps/parsers/services.py b/src/apps/parsers/services.py index 87bd745..4b16c1f 100644 --- a/src/apps/parsers/services.py +++ b/src/apps/parsers/services.py @@ -34,7 +34,7 @@ from apps.parsers.models import ( ProcurementRecord, Proxy, ) -from apps.registers.models import Organization +from registers.models import Organization from django.conf import settings from django.db import IntegrityError, transaction from django.db.models import Q diff --git a/src/apps/registers/pagination.py b/src/apps/registers/pagination.py deleted file mode 100644 index 42de890..0000000 --- a/src/apps/registers/pagination.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Pagination helpers for registers app.""" - -from apps.core.pagination import StandardPagination - - -class RegistersPagination(StandardPagination): - """ - Пагинация для реестров с поддержкой meta.actual_date. - - Если view выставляет атрибут `actual_date_meta`, он будет добавлен - в метаданные ответа. - """ - - _view = None - - def paginate_queryset(self, queryset, request, view=None): - self._view = view - return super().paginate_queryset(queryset, request, view=view) - - def get_paginated_response(self, data): - response = super().get_paginated_response(data) - - actual_date = getattr(self._view, "actual_date_meta", None) - if actual_date: - meta = response.data.get("meta") or {} - meta["actual_date"] = actual_date.isoformat() - response.data["meta"] = meta - - return response diff --git a/src/core/api_v1_urls.py b/src/core/api_v1_urls.py index 62e7b1c..53a0d54 100644 --- a/src/core/api_v1_urls.py +++ b/src/core/api_v1_urls.py @@ -33,7 +33,7 @@ from apps.parsers.urls import ( system_urlpatterns, zakupki_urlpatterns, ) -from apps.registers.urls import registers_urlpatterns +from registers.urls import registers_urlpatterns from django.urls import include, path app_name = "api_v1" @@ -47,7 +47,7 @@ jobs_urlpatterns = [ urlpatterns = [ # Аутентификация и пользователи - path("users/", include("apps.user.urls")), + path("users/", include("user.urls")), # Фоновые задачи path("jobs/", include((jobs_urlpatterns, "jobs"))), # Парсеры - Минпромторг diff --git a/src/apps/registers/__init__.py b/src/registers/__init__.py similarity index 100% rename from src/apps/registers/__init__.py rename to src/registers/__init__.py diff --git a/src/apps/registers/admin.py b/src/registers/admin.py similarity index 93% rename from src/apps/registers/admin.py rename to src/registers/admin.py index 4c89ec6..f7b2026 100644 --- a/src/apps/registers/admin.py +++ b/src/registers/admin.py @@ -1,13 +1,13 @@ """Admin configuration for registers app.""" -from apps.registers.models import ( +from registers.models import ( Organization, Register, RegisterUpload, RegistryMembershipPeriod, ) -from apps.registers.serializers import RegisterFileUploadSerializer -from apps.registers.services import RegisterImportError, RegisterImportService +from registers.serializers import RegisterFileUploadSerializer +from registers.services import RegisterImportError, RegisterImportService from django.contrib import admin, messages from django.shortcuts import redirect from django.template.response import TemplateResponse @@ -25,7 +25,7 @@ class RegisterAdmin(admin.ModelAdmin): ordering = ["name"] def active_organizations_count(self, obj): - return obj.membership_periods.filter(ended_at__isnull=True).count() + return obj.membership_periods.count() active_organizations_count.short_description = "Активных организаций" @@ -123,7 +123,6 @@ class RegisterUploadAdmin(admin.ModelAdmin): result = RegisterImportService.sync_registry_memberships( registry=serializer.validated_data["registry"], uploaded_file=uploaded_file, - actual_date=serializer.validated_data.get("actual_date"), uploaded_by=request.user, ) except RegisterImportError as exc: @@ -222,12 +221,9 @@ class RegistryMembershipPeriodAdmin(admin.ModelAdmin): list_display = [ "registry", "organization_short", - "started_at", - "ended_at", - "started_by_upload", - "ended_by_upload", + "created_at", ] - list_filter = ["registry", "started_at", "ended_at"] + list_filter = ["registry"] search_fields = [ "registry__name", "organization__pn_name", @@ -235,7 +231,7 @@ class RegistryMembershipPeriodAdmin(admin.ModelAdmin): "organization__mn_inn", ] readonly_fields = ["created_at", "updated_at"] - ordering = ["registry__name", "-started_at"] + ordering = ["registry__name", "organization__pn_name"] def organization_short(self, obj): name = obj.organization.pn_name or "" diff --git a/src/apps/registers/apps.py b/src/registers/apps.py similarity index 92% rename from src/apps/registers/apps.py rename to src/registers/apps.py index 869b5b6..7146cca 100644 --- a/src/apps/registers/apps.py +++ b/src/registers/apps.py @@ -5,7 +5,7 @@ class RegistersConfig(AppConfig): """Конфигурация приложения реестров.""" default_auto_field = "django.db.models.BigAutoField" - name = "apps.registers" + name = "registers" verbose_name = "Реестры организаций" def ready(self): diff --git a/src/apps/registers/management/__init__.py b/src/registers/management/__init__.py similarity index 100% rename from src/apps/registers/management/__init__.py rename to src/registers/management/__init__.py diff --git a/src/apps/registers/management/commands/__init__.py b/src/registers/management/commands/__init__.py similarity index 100% rename from src/apps/registers/management/commands/__init__.py rename to src/registers/management/commands/__init__.py diff --git a/src/apps/registers/management/commands/generate_test_data.py b/src/registers/management/commands/generate_test_data.py similarity index 97% rename from src/apps/registers/management/commands/generate_test_data.py rename to src/registers/management/commands/generate_test_data.py index 895abda..117f64b 100644 --- a/src/apps/registers/management/commands/generate_test_data.py +++ b/src/registers/management/commands/generate_test_data.py @@ -34,7 +34,7 @@ class Command(BaseAppCommand): "--registers-count", type=int, default=5, - help="Количество записей в справочнике реестров (apps.registers)", + help="Количество записей в справочнике реестров (registers)", ) def execute_command(self, *args: Any, **options: Any) -> str: @@ -83,11 +83,10 @@ class Command(BaseAppCommand): mn_inn=1_000_000_000 + (run_seed * 10_000 + org_serial) % 8_000_000_000, ) - upload = register_upload_factory.create(registry=register) + register_upload_factory.create(registry=register) registry_membership_period_factory.create( registry=register, organization=organization, - started_by_upload=upload, ) self.log_info("Создание тестовых данных parser-реестров...") @@ -163,7 +162,7 @@ class Command(BaseAppCommand): Command._ensure_project_root_on_path() try: - registers_factories = import_module("tests.apps.registers.factories") + registers_factories = import_module("tests.registers.factories") parsers_factories = import_module("tests.apps.parsers.factories") except ModuleNotFoundError as exc: raise CommandError( diff --git a/src/apps/registers/migrations/0001_initial.py b/src/registers/migrations/0001_initial.py similarity index 100% rename from src/apps/registers/migrations/0001_initial.py rename to src/registers/migrations/0001_initial.py diff --git a/src/apps/registers/migrations/0002_auto_20260304_1038.py b/src/registers/migrations/0002_auto_20260304_1038.py similarity index 100% rename from src/apps/registers/migrations/0002_auto_20260304_1038.py rename to src/registers/migrations/0002_auto_20260304_1038.py diff --git a/src/apps/registers/migrations/0003_add_unique_active_membership_period.py b/src/registers/migrations/0003_add_unique_active_membership_period.py similarity index 100% rename from src/apps/registers/migrations/0003_add_unique_active_membership_period.py rename to src/registers/migrations/0003_add_unique_active_membership_period.py diff --git a/src/apps/registers/migrations/0004_seed_default_registers.py b/src/registers/migrations/0004_seed_default_registers.py similarity index 100% rename from src/apps/registers/migrations/0004_seed_default_registers.py rename to src/registers/migrations/0004_seed_default_registers.py diff --git a/src/apps/registers/migrations/0005_seed_additional_registers.py b/src/registers/migrations/0005_seed_additional_registers.py similarity index 100% rename from src/apps/registers/migrations/0005_seed_additional_registers.py rename to src/registers/migrations/0005_seed_additional_registers.py diff --git a/src/registers/migrations/0006_flat_membership_period.py b/src/registers/migrations/0006_flat_membership_period.py new file mode 100644 index 0000000..da6b989 --- /dev/null +++ b/src/registers/migrations/0006_flat_membership_period.py @@ -0,0 +1,47 @@ +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("registers", "0005_seed_additional_registers"), + ] + + operations = [ + migrations.RemoveConstraint( + model_name="registrymembershipperiod", + name="unique_active_membership_period", + ), + migrations.RemoveConstraint( + model_name="registrymembershipperiod", + name="check_membership_period_dates", + ), + migrations.RemoveIndex( + model_name="registrymembershipperiod", + name="registers_m_registr_3292a6_idx", + ), + migrations.RemoveIndex( + model_name="registrymembershipperiod", + name="registers_m_registr_edbdd9_idx", + ), + migrations.RemoveIndex( + model_name="registrymembershipperiod", + name="registers_m_organiz_138ba3_idx", + ), + migrations.RemoveField( + model_name="registrymembershipperiod", + name="started_at", + ), + migrations.RemoveField( + model_name="registrymembershipperiod", + name="ended_at", + ), + migrations.RemoveField( + model_name="registrymembershipperiod", + name="started_by_upload", + ), + migrations.RemoveField( + model_name="registrymembershipperiod", + name="ended_by_upload", + ), + ] diff --git a/src/apps/registers/migrations/__init__.py b/src/registers/migrations/__init__.py similarity index 100% rename from src/apps/registers/migrations/__init__.py rename to src/registers/migrations/__init__.py diff --git a/src/apps/registers/models.py b/src/registers/models.py similarity index 74% rename from src/apps/registers/models.py rename to src/registers/models.py index 3bb3d97..5124271 100644 --- a/src/apps/registers/models.py +++ b/src/registers/models.py @@ -4,7 +4,6 @@ from apps.core.mixins import TimestampMixin, UUIDPrimaryKeyMixin from django.conf import settings from django.core.validators import RegexValidator from django.db import models -from django.db.models import Q from django.utils.translation import gettext_lazy as _ @@ -87,6 +86,10 @@ class Organization(TimestampMixin, models.Model): class RegisterUpload(TimestampMixin, models.Model): """Факт загрузки снимка реестра на конкретную дату актуальности.""" + class ImportStatus(models.TextChoices): + SUCCESS = "success", "Успешная" + FAILED = "failed", "Ошибка" + registry = models.ForeignKey( Register, on_delete=models.CASCADE, @@ -109,6 +112,20 @@ class RegisterUpload(TimestampMixin, models.Model): db_index=True, help_text=_("SHA-256 хеш загруженного файла"), ) + import_status = models.CharField( + _("статус импорта"), + max_length=16, + choices=ImportStatus.choices, + default=ImportStatus.SUCCESS, + db_index=True, + help_text=_("Результат попытки импорта файла в реестр"), + ) + import_error = models.TextField( + _("текст ошибки импорта"), + blank=True, + null=True, + help_text=_("Детализация ошибки, если импорт завершился неуспешно"), + ) rows_count = models.PositiveIntegerField( _("количество строк"), default=0, @@ -138,7 +155,7 @@ class RegisterUpload(TimestampMixin, models.Model): class RegistryMembershipPeriod(TimestampMixin, models.Model): - """Период присутствия организации в конкретном реестре.""" + """Текущая принадлежность организации к реестру.""" registry = models.ForeignKey( Register, @@ -152,58 +169,22 @@ class RegistryMembershipPeriod(TimestampMixin, models.Model): related_name="membership_periods", verbose_name=_("организация"), ) - started_at = models.DateField( - _("дата входа"), - db_index=True, - help_text=_("Дата, с которой организация входит в реестр"), - ) - ended_at = models.DateField( - _("дата выхода"), - null=True, - blank=True, - db_index=True, - help_text=_("Дата, с которой организация больше не входит в реестр"), - ) - started_by_upload = models.ForeignKey( - RegisterUpload, - on_delete=models.PROTECT, - related_name="started_periods", - verbose_name=_("загрузка входа"), - ) - ended_by_upload = models.ForeignKey( - RegisterUpload, - on_delete=models.PROTECT, - null=True, - blank=True, - related_name="ended_periods", - verbose_name=_("загрузка выхода"), - ) class Meta: db_table = "registers_membership_period" - verbose_name = _("период участия") - verbose_name_plural = _("периоды участия") - ordering = ["-started_at", "registry_id"] + verbose_name = _("участие в реестре") + verbose_name_plural = _("участия в реестрах") + ordering = ["registry_id", "organization_id"] indexes = [ - models.Index(fields=["registry", "started_at"]), - models.Index(fields=["registry", "ended_at"]), - models.Index(fields=["organization", "started_at"]), + models.Index(fields=["registry"]), + models.Index(fields=["organization"]), ] constraints = [ - models.CheckConstraint( - check=Q(ended_at__isnull=True) - | Q(ended_at__gte=models.F("started_at")), - name="check_membership_period_dates", - ), models.UniqueConstraint( fields=["registry", "organization"], - condition=Q(ended_at__isnull=True), - name="unique_active_membership_period", + name="unique_membership", ), ] def __str__(self) -> str: - return ( - f"{self.registry.name}: {self.organization.pn_name[:40]} " - f"[{self.started_at} - {self.ended_at or '...'}]" - ) + return f"{self.registry.name}: {self.organization.pn_name[:40]}" diff --git a/src/registers/pagination.py b/src/registers/pagination.py new file mode 100644 index 0000000..e361ccb --- /dev/null +++ b/src/registers/pagination.py @@ -0,0 +1,7 @@ +"""Pagination helpers for registers app.""" + +from apps.core.pagination import StandardPagination + + +class RegistersPagination(StandardPagination): + """Пагинация для реестров.""" diff --git a/src/apps/registers/serializers.py b/src/registers/serializers.py similarity index 87% rename from src/apps/registers/serializers.py rename to src/registers/serializers.py index 965218a..fdf64cf 100644 --- a/src/apps/registers/serializers.py +++ b/src/registers/serializers.py @@ -1,6 +1,6 @@ """Сериализаторы для API реестров.""" -from apps.registers.models import ( +from registers.models import ( Organization, Register, RegistryMembershipPeriod, @@ -42,8 +42,6 @@ class RegistryMembershipPeriodSerializer(serializers.ModelSerializer): "id", "registry_id", "registry_name", - "started_at", - "ended_at", ] read_only_fields = fields @@ -84,7 +82,6 @@ class RegisterFileUploadSerializer(serializers.Serializer): registry = serializers.PrimaryKeyRelatedField(queryset=Register.objects.all()) file = serializers.FileField() - actual_date = serializers.DateField(required=False) def validate_file(self, value): if not value.name.lower().endswith(".xlsx"): @@ -99,22 +96,12 @@ class OrganizationListQuerySerializer(serializers.Serializer): queryset=Register.objects.all(), required=False, ) - actual_date = serializers.DateField(required=False) search = serializers.CharField(required=False, allow_blank=True) mn_ogrn = serializers.IntegerField(required=False, min_value=0) mn_inn = serializers.IntegerField(required=False, min_value=0) in_kpp = serializers.IntegerField(required=False, min_value=0) mn_okpo = serializers.CharField(required=False, allow_blank=False) - def validate(self, attrs): - if attrs.get("actual_date") and not attrs.get("registry"): - raise serializers.ValidationError( - { - "actual_date": "Параметр actual_date допустим только вместе с registry" - } - ) - return attrs - def validate_mn_okpo(self, value): if not value.isdigit(): raise serializers.ValidationError("mn_okpo должен содержать только цифры") @@ -124,7 +111,6 @@ class OrganizationListQuerySerializer(serializers.Serializer): class RegistryOrganizationListQuerySerializer(serializers.Serializer): """Сериализатор query-параметров списка организаций конкретного реестра.""" - actual_date = serializers.DateField(required=False) search = serializers.CharField(required=False, allow_blank=True) mn_ogrn = serializers.IntegerField(required=False, min_value=0) mn_inn = serializers.IntegerField(required=False, min_value=0) diff --git a/src/apps/registers/services.py b/src/registers/services.py similarity index 63% rename from src/apps/registers/services.py rename to src/registers/services.py index 6bb73aa..5c3e1e2 100644 --- a/src/apps/registers/services.py +++ b/src/registers/services.py @@ -4,9 +4,8 @@ from __future__ import annotations import hashlib from dataclasses import dataclass -from datetime import date -from apps.registers.models import ( +from registers.models import ( Organization, Register, RegisterUpload, @@ -46,75 +45,103 @@ class RegisterImportService: *, registry: Register, uploaded_file, - actual_date: date | None = None, uploaded_by=None, ) -> dict[str, int | str]: """ - Импортировать снимок реестра на дату актуальности. + Обновить текущее состояние реестра целиком из загруженного файла. Алгоритм: 1. Парсим файл и upsert-им канонические организации. - 2. Закрываем активные периоды организаций, которых нет в новом снимке. - 3. Открываем периоды для организаций, которых не было среди активных. + 2. Полностью заменяем текущее состояние реестра в соответствии + с выгруженным списком организаций. """ - snapshot_date = actual_date or timezone.localdate() - - parsed_rows = cls.parse_xlsx(uploaded_file) - rows = cls._ensure_unique_identities(parsed_rows) file_hash = cls._calculate_file_hash(uploaded_file) - cls._validate_snapshot_date(registry=registry, snapshot_date=snapshot_date) - upload = RegisterUpload.objects.create( registry=registry, - actual_date=snapshot_date, + actual_date=timezone.localdate(), file_name=uploaded_file.name, file_hash=file_hash, - rows_count=len(rows), + rows_count=0, uploaded_by=uploaded_by, ) - ( - snapshot_org_ids, - organizations_created, - organizations_updated, - ) = cls._upsert_organizations(rows) + try: + rows = cls._ensure_unique_identities(cls.parse_xlsx(uploaded_file)) - active_by_org = cls._get_active_periods_by_org(registry) - active_org_ids = set(active_by_org.keys()) + with transaction.atomic(): + ( + snapshot_org_ids, + organizations_created, + organizations_updated, + ) = cls._upsert_organizations(rows) - closed_periods = cls._close_missing_periods( - active_by_org=active_by_org, - snapshot_org_ids=snapshot_org_ids, - snapshot_date=snapshot_date, - upload=upload, - ) + existing_org_ids = set( + RegistryMembershipPeriod.objects.filter(registry=registry).values_list( + "organization_id", flat=True + ) + ) + snapshot_org_ids_set = set(snapshot_org_ids) - opened_periods = cls._open_new_periods( - registry=registry, - snapshot_org_ids=snapshot_org_ids, - active_org_ids=active_org_ids, - snapshot_date=snapshot_date, - upload=upload, - ) + to_remove_org_ids = existing_org_ids - snapshot_org_ids_set + to_add_org_ids = snapshot_org_ids_set - existing_org_ids - active_periods_count = RegistryMembershipPeriod.objects.filter( - registry=registry, - ended_at__isnull=True, - ).count() + if to_remove_org_ids: + RegistryMembershipPeriod.objects.filter( + registry=registry, + organization_id__in=to_remove_org_ids, + ).delete() - return { - "upload_id": upload.id, - "registry_id": str(registry.id), - "registry_name": registry.name, - "actual_date": snapshot_date.isoformat(), - "rows_in_file": len(rows), - "organizations_created": organizations_created, - "organizations_updated": organizations_updated, - "opened_periods": opened_periods, - "closed_periods": closed_periods, - "active_periods": active_periods_count, - } + if to_add_org_ids: + RegistryMembershipPeriod.objects.bulk_create( + [ + RegistryMembershipPeriod( + registry=registry, + organization_id=organization_id, + ) + for organization_id in to_add_org_ids + ], + batch_size=1000, + ignore_conflicts=True, + ) + + active_memberships_count = RegistryMembershipPeriod.objects.filter( + registry=registry + ).count() + + upload.rows_count = len(rows) + upload.import_status = RegisterUpload.ImportStatus.SUCCESS + upload.import_error = None + upload.save( + update_fields=[ + "rows_count", + "import_status", + "import_error", + "updated_at", + ] + ) + + return { + "upload_id": upload.id, + "registry_id": str(registry.id), + "registry_name": registry.name, + "rows_in_file": len(rows), + "organizations_created": organizations_created, + "organizations_updated": organizations_updated, + "active_memberships": active_memberships_count, + } + except RegisterImportError as exc: + upload.import_status = RegisterUpload.ImportStatus.FAILED + upload.import_error = str(exc) + upload.save( + update_fields=[ + "import_status", + "import_error", + "rows_count", + "updated_at", + ] + ) + raise @classmethod def _upsert_organizations( @@ -176,138 +203,27 @@ class RegisterImportService: organization.save(update_fields=update_fields + ["updated_at"]) return True - @classmethod - def _get_active_periods_by_org( - cls, - registry: Register, - ) -> dict[int, RegistryMembershipPeriod]: - active_periods = ( - RegistryMembershipPeriod.objects.select_for_update() - .filter(registry=registry, ended_at__isnull=True) - .only("id", "organization_id", "started_at") - ) - return {period.organization_id: period for period in active_periods} - - @classmethod - def _close_missing_periods( - cls, - *, - active_by_org: dict[int, RegistryMembershipPeriod], - snapshot_org_ids: set[int], - snapshot_date: date, - upload: RegisterUpload, - ) -> int: - close_by_date_ids: list[int] = [] - close_by_delete_ids: list[int] = [] - - for organization_id, period in active_by_org.items(): - if organization_id in snapshot_org_ids: - continue - - # Если загружали этот же snapshot_date повторно, удаляем нулевой период. - if period.started_at == snapshot_date: - close_by_delete_ids.append(period.id) - else: - close_by_date_ids.append(period.id) - - closed_periods = 0 - - if close_by_date_ids: - closed_periods += RegistryMembershipPeriod.objects.filter( - id__in=close_by_date_ids - ).update( - ended_at=snapshot_date, - ended_by_upload=upload, - ) - - if close_by_delete_ids: - deleted_count, _ = RegistryMembershipPeriod.objects.filter( - id__in=close_by_delete_ids - ).delete() - closed_periods += deleted_count - - return closed_periods - - @classmethod - def _open_new_periods( - cls, - *, - registry: Register, - snapshot_org_ids: set[int], - active_org_ids: set[int], - snapshot_date: date, - upload: RegisterUpload, - ) -> int: - to_open_org_ids = snapshot_org_ids - active_org_ids - if not to_open_org_ids: - return 0 - - RegistryMembershipPeriod.objects.bulk_create( - [ - RegistryMembershipPeriod( - registry=registry, - organization_id=organization_id, - started_at=snapshot_date, - started_by_upload=upload, - ) - for organization_id in to_open_org_ids - ], - batch_size=1000, - ignore_conflicts=True, - ) - - return RegistryMembershipPeriod.objects.filter( - started_by_upload=upload, - ).count() - - @classmethod - def resolve_actual_date( - cls, - *, - registry: Register, - requested_date: date | None, - ) -> date: - """Вернуть effective дату среза: запрошенную или последнюю доступную.""" - if requested_date: - return requested_date - - latest_upload = ( - RegisterUpload.objects.filter(registry=registry) - .order_by("-actual_date", "-id") - .first() - ) - - if latest_upload: - return latest_upload.actual_date - - return timezone.localdate() - @classmethod def get_organizations_queryset( cls, *, registry: Register | None = None, - actual_date: date | None = None, search: str = "", mn_ogrn: int | None = None, mn_inn: int | None = None, in_kpp: int | None = None, mn_okpo: str | None = None, - ) -> tuple: - """Получить queryset организаций с учетом фильтров и среза по реестру.""" + ): + """Получить queryset организаций с учетом фильтров по текущему состоянию.""" queryset = Organization.objects.all().order_by("pn_name") - resolved_actual_date = None if registry: - resolved_actual_date = cls.resolve_actual_date( - registry=registry, - requested_date=actual_date, - ) - queryset = cls._filter_organizations_active_in_registry( + queryset = cls._filter_organizations_in_registry( queryset=queryset, registry=registry, - actual_date=resolved_actual_date, - ).distinct() + ) + + queryset = queryset.distinct() queryset = cls._apply_exact_filters( queryset, @@ -318,30 +234,23 @@ class RegisterImportService: ) queryset = cls._apply_search(queryset, search.strip()) - return queryset, resolved_actual_date + return queryset @classmethod def get_registry_organizations_queryset( cls, *, registry: Register, - actual_date: date | None = None, search: str = "", mn_ogrn: int | None = None, mn_inn: int | None = None, in_kpp: int | None = None, mn_okpo: str | None = None, - ) -> tuple: - """Получить queryset организаций конкретного реестра на дату.""" - resolved_actual_date = cls.resolve_actual_date( - registry=registry, - requested_date=actual_date, - ) - - queryset = cls._filter_organizations_active_in_registry( + ): + """Получить queryset организаций конкретного реестра.""" + queryset = cls._filter_organizations_in_registry( queryset=Organization.objects.all().order_by("pn_name"), registry=registry, - actual_date=resolved_actual_date, ).distinct() queryset = cls._apply_exact_filters( queryset, @@ -352,23 +261,16 @@ class RegisterImportService: ) queryset = cls._apply_search(queryset, search.strip()) - return queryset, resolved_actual_date + return queryset @classmethod - def _filter_organizations_active_in_registry( + def _filter_organizations_in_registry( cls, *, queryset, registry: Register, - actual_date: date, ): - return queryset.filter( - membership_periods__registry=registry, - membership_periods__started_at__lte=actual_date, - ).filter( - Q(membership_periods__ended_at__isnull=True) - | Q(membership_periods__ended_at__gt=actual_date) - ) + return queryset.filter(membership_periods__registry=registry) @classmethod def _apply_exact_filters( @@ -471,21 +373,6 @@ class RegisterImportService: finally: workbook.close() - @classmethod - def _validate_snapshot_date( - cls, *, registry: Register, snapshot_date: date - ) -> None: - latest_upload = ( - RegisterUpload.objects.filter(registry=registry) - .order_by("-actual_date", "-id") - .first() - ) - if latest_upload and snapshot_date < latest_upload.actual_date: - raise RegisterImportError( - "Дата актуальности не может быть раньше последней загрузки " - f"({latest_upload.actual_date.isoformat()})" - ) - @classmethod def _calculate_file_hash(cls, uploaded_file) -> str: uploaded_file.seek(0) diff --git a/src/apps/registers/signals.py b/src/registers/signals.py similarity index 95% rename from src/apps/registers/signals.py rename to src/registers/signals.py index 7afb645..cbe081c 100644 --- a/src/apps/registers/signals.py +++ b/src/registers/signals.py @@ -18,7 +18,7 @@ DEFAULT_REGISTER_NAMES = ( @receiver(post_migrate) def seed_default_registers(sender, **kwargs): """Create default registries on fresh environments.""" - if sender.name != "apps.registers": + if sender.name != "registers": return Register = apps.get_model("registers", "Register") diff --git a/src/apps/registers/urls.py b/src/registers/urls.py similarity index 95% rename from src/apps/registers/urls.py rename to src/registers/urls.py index 61a07b9..efe8833 100644 --- a/src/apps/registers/urls.py +++ b/src/registers/urls.py @@ -1,6 +1,6 @@ """URL конфигурация для приложения реестров.""" -from apps.registers.views import ( +from registers.views import ( OrganizationViewSet, RegisterUploadView, RegisterViewSet, diff --git a/src/apps/registers/views.py b/src/registers/views.py similarity index 80% rename from src/apps/registers/views.py rename to src/registers/views.py index 1727f9d..f6a0962 100644 --- a/src/apps/registers/views.py +++ b/src/registers/views.py @@ -2,12 +2,10 @@ from __future__ import annotations -from datetime import date - from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag -from apps.registers.models import Organization, Register -from apps.registers.pagination import RegistersPagination -from apps.registers.serializers import ( +from registers.models import Organization, Register +from registers.pagination import RegistersPagination +from registers.serializers import ( OrganizationDetailSerializer, OrganizationListQuerySerializer, OrganizationSerializer, @@ -17,7 +15,7 @@ from apps.registers.serializers import ( RegisterUploadSuccessSerializer, RegistryOrganizationListQuerySerializer, ) -from apps.registers.services import RegisterImportError, RegisterImportService +from registers.services import RegisterImportError, RegisterImportService from django.shortcuts import get_object_or_404 from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -73,14 +71,13 @@ class OrganizationViewSet(ReadOnlyModelViewSet): API для просмотра организаций. Поддерживает глобальный поиск по бизнес-полям, а также фильтрацию - по `registry` и `actual_date` (срез организаций в реестре на дату). + по `registry`. """ queryset = Organization.objects.all().order_by("pn_name") serializer_class = OrganizationSerializer permission_classes = [IsAuthenticated] pagination_class = RegistersPagination - actual_date_meta: date | None = None def get_serializer_class(self): if self.action == "retrieve": @@ -88,8 +85,6 @@ class OrganizationViewSet(ReadOnlyModelViewSet): return OrganizationSerializer def get_queryset(self): - self.actual_date_meta = None - if self.action == "retrieve": return ( Organization.objects.all() @@ -102,10 +97,7 @@ class OrganizationViewSet(ReadOnlyModelViewSet): ) params_serializer.is_valid(raise_exception=True) - ( - queryset, - self.actual_date_meta, - ) = RegisterImportService.get_organizations_queryset( + queryset = RegisterImportService.get_organizations_queryset( **params_serializer.validated_data ) @@ -117,8 +109,7 @@ class OrganizationViewSet(ReadOnlyModelViewSet): operation_description=( "Возвращает список организаций.\n" "Поддерживает фильтрацию по: mn_ogrn, mn_inn, in_kpp, mn_okpo.\n" - "Поддерживает поиск (search) по: pn_name, mn_ogrn, mn_inn, in_kpp, mn_okpo.\n" - "Опционально можно указать registry и actual_date для среза реестра на дату." + "Поддерживает поиск (search) по: pn_name, mn_ogrn, mn_inn, in_kpp, mn_okpo." ), manual_parameters=[ openapi.Parameter( @@ -129,14 +120,6 @@ class OrganizationViewSet(ReadOnlyModelViewSet): required=False, description="UUID реестра для среза", ), - openapi.Parameter( - name="actual_date", - in_=openapi.IN_QUERY, - type=openapi.TYPE_STRING, - format=openapi.FORMAT_DATE, - required=False, - description="Дата актуальности среза (YYYY-MM-DD)", - ), openapi.Parameter( name="search", in_=openapi.IN_QUERY, @@ -195,18 +178,16 @@ class OrganizationViewSet(ReadOnlyModelViewSet): class RegistryOrganizationListView(ListAPIView): - """API списка организаций конкретного реестра на дату актуальности.""" + """API списка организаций конкретного реестра.""" serializer_class = OrganizationSerializer permission_classes = [IsAuthenticated] pagination_class = RegistersPagination - actual_date_meta: date | None = None def _get_registry(self) -> Register: return get_object_or_404(Register, id=self.kwargs["registry_id"]) def get_queryset(self): - self.actual_date_meta = None registry = self._get_registry() params_serializer = RegistryOrganizationListQuerySerializer( @@ -214,10 +195,7 @@ class RegistryOrganizationListView(ListAPIView): ) params_serializer.is_valid(raise_exception=True) - ( - queryset, - self.actual_date_meta, - ) = RegisterImportService.get_registry_organizations_queryset( + queryset = RegisterImportService.get_registry_organizations_queryset( registry=registry, **params_serializer.validated_data, ) @@ -229,18 +207,10 @@ class RegistryOrganizationListView(ListAPIView): operation_summary="Список организаций реестра", operation_description=( "Возвращает список организаций только для указанного реестра.\n" - "Опциональный параметр actual_date задаёт срез на дату.\n" - "Если actual_date не передан, используется последняя доступная дата загрузки." + "Если registry_id валиден, список возвращает только организации " + "этого реестра." ), manual_parameters=[ - openapi.Parameter( - name="actual_date", - in_=openapi.IN_QUERY, - type=openapi.TYPE_STRING, - format=openapi.FORMAT_DATE, - required=False, - description="Дата актуальности среза (YYYY-MM-DD)", - ), openapi.Parameter( name="search", in_=openapi.IN_QUERY, @@ -298,9 +268,7 @@ class RegisterUploadView(APIView): operation_description=( "Загружает Excel (.xlsx) с организациями в выбранный реестр.\n" "Требуемые колонки: pn_name, mn_ogrn, mn_inn, mn_okpo.\n" - "Опциональная колонка: in_kpp.\n" - "actual_date задаёт дату актуальности среза.\n" - "Если организация исчезла из следующего среза, для неё закрывается период участия." + "Опциональная колонка: in_kpp." ), manual_parameters=[ openapi.Parameter( @@ -311,14 +279,6 @@ class RegisterUploadView(APIView): required=True, description="UUID выбранного реестра", ), - openapi.Parameter( - name="actual_date", - in_=openapi.IN_FORM, - type=openapi.TYPE_STRING, - format=openapi.FORMAT_DATE, - required=False, - description="Дата актуальности (YYYY-MM-DD)", - ), openapi.Parameter( name="file", in_=openapi.IN_FORM, @@ -340,13 +300,11 @@ class RegisterUploadView(APIView): registry = serializer.validated_data["registry"] uploaded_file = serializer.validated_data["file"] - actual_date = serializer.validated_data.get("actual_date") try: RegisterImportService.sync_registry_memberships( registry=registry, uploaded_file=uploaded_file, - actual_date=actual_date, uploaded_by=request.user, ) except RegisterImportError as exc: diff --git a/src/settings/base.py b/src/settings/base.py index 0527911..1198f95 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -46,9 +46,9 @@ INSTALLED_APPS = [ "drf_yasg", # Local apps "apps.core", - "apps.user", + "user", "apps.parsers", - "apps.registers", + "registers", "apps.exchange", "apps.backups", ] diff --git a/src/apps/user/__init__.py b/src/user/__init__.py similarity index 100% rename from src/apps/user/__init__.py rename to src/user/__init__.py diff --git a/src/apps/user/admin.py b/src/user/admin.py similarity index 99% rename from src/apps/user/admin.py rename to src/user/admin.py index 636dd61..4cf637a 100644 --- a/src/apps/user/admin.py +++ b/src/user/admin.py @@ -2,7 +2,7 @@ Admin configuration for user app. """ -from apps.user.models import Profile, User +from user.models import Profile, User from django.contrib import admin from django.contrib.auth.admin import UserAdmin as BaseUserAdmin from django.utils.html import format_html diff --git a/src/apps/user/apps.py b/src/user/apps.py similarity index 74% rename from src/apps/user/apps.py rename to src/user/apps.py index fd83ada..a33b89b 100644 --- a/src/apps/user/apps.py +++ b/src/user/apps.py @@ -3,8 +3,8 @@ from django.apps import AppConfig class UserConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" - name = "apps.user" + name = "user" verbose_name = "Пользователи" def ready(self): - import apps.user.signals # noqa + import user.signals # noqa diff --git a/src/apps/user/migrations/0001_initial.py b/src/user/migrations/0001_initial.py similarity index 100% rename from src/apps/user/migrations/0001_initial.py rename to src/user/migrations/0001_initial.py diff --git a/src/apps/user/migrations/0002_remove_firstname_lastname.py b/src/user/migrations/0002_remove_firstname_lastname.py similarity index 100% rename from src/apps/user/migrations/0002_remove_firstname_lastname.py rename to src/user/migrations/0002_remove_firstname_lastname.py diff --git a/src/apps/user/migrations/0003_alter_user_groups.py b/src/user/migrations/0003_alter_user_groups.py similarity index 100% rename from src/apps/user/migrations/0003_alter_user_groups.py rename to src/user/migrations/0003_alter_user_groups.py diff --git a/src/apps/user/migrations/0004_alter_user_groups.py b/src/user/migrations/0004_alter_user_groups.py similarity index 100% rename from src/apps/user/migrations/0004_alter_user_groups.py rename to src/user/migrations/0004_alter_user_groups.py diff --git a/src/apps/user/migrations/0005_create_default_admin_superuser.py b/src/user/migrations/0005_create_default_admin_superuser.py similarity index 100% rename from src/apps/user/migrations/0005_create_default_admin_superuser.py rename to src/user/migrations/0005_create_default_admin_superuser.py diff --git a/src/apps/user/migrations/0006_create_default_role_groups.py b/src/user/migrations/0006_create_default_role_groups.py similarity index 100% rename from src/apps/user/migrations/0006_create_default_role_groups.py rename to src/user/migrations/0006_create_default_role_groups.py diff --git a/src/apps/user/migrations/0007_profile_middle_name.py b/src/user/migrations/0007_profile_middle_name.py similarity index 100% rename from src/apps/user/migrations/0007_profile_middle_name.py rename to src/user/migrations/0007_profile_middle_name.py diff --git a/src/apps/user/migrations/0008_alter_user_groups.py b/src/user/migrations/0008_alter_user_groups.py similarity index 100% rename from src/apps/user/migrations/0008_alter_user_groups.py rename to src/user/migrations/0008_alter_user_groups.py diff --git a/src/apps/user/migrations/0009_alter_user_groups.py b/src/user/migrations/0009_alter_user_groups.py similarity index 100% rename from src/apps/user/migrations/0009_alter_user_groups.py rename to src/user/migrations/0009_alter_user_groups.py diff --git a/src/apps/user/migrations/0010_profile_names_required.py b/src/user/migrations/0010_profile_names_required.py similarity index 100% rename from src/apps/user/migrations/0010_profile_names_required.py rename to src/user/migrations/0010_profile_names_required.py diff --git a/src/apps/user/migrations/__init__.py b/src/user/migrations/__init__.py similarity index 100% rename from src/apps/user/migrations/__init__.py rename to src/user/migrations/__init__.py diff --git a/src/apps/user/models.py b/src/user/models.py similarity index 100% rename from src/apps/user/models.py rename to src/user/models.py diff --git a/src/apps/user/serializers.py b/src/user/serializers.py similarity index 100% rename from src/apps/user/serializers.py rename to src/user/serializers.py diff --git a/src/apps/user/services.py b/src/user/services.py similarity index 100% rename from src/apps/user/services.py rename to src/user/services.py diff --git a/src/apps/user/signals.py b/src/user/signals.py similarity index 100% rename from src/apps/user/signals.py rename to src/user/signals.py diff --git a/src/apps/user/urls.py b/src/user/urls.py similarity index 100% rename from src/apps/user/urls.py rename to src/user/urls.py diff --git a/src/apps/user/views.py b/src/user/views.py similarity index 100% rename from src/apps/user/views.py rename to src/user/views.py