feat(organizations): migrate source storage to polymorphic records
This commit is contained in:
@@ -5,14 +5,44 @@ import uuid
|
||||
from django.db import models
|
||||
from django.db.models import Q
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from polymorphic.models import PolymorphicModel
|
||||
|
||||
from organizations.data_sources import snapshot_data_source_summary
|
||||
from organizations.name_normalization import normalize_organization_name
|
||||
|
||||
|
||||
class SourceGroup(models.TextChoices):
|
||||
"""Product-level organization source groups."""
|
||||
|
||||
FINANCIAL_INDICATORS = "financial_indicators", _("Финансово-экономические показатели")
|
||||
GOVERNMENT_PROCUREMENTS = "government_procurements", _("Государственные закупки")
|
||||
INDUSTRIAL_PRODUCTION = "industrial_production", _("Производители и продукция России")
|
||||
PLANNED_INSPECTIONS = "planned_inspections", _("Плановые проверки")
|
||||
BANKRUPTCY = "bankruptcy", _("Сведения о процедурах банкротства")
|
||||
DEFENSE_SUPPLIERS = "defense_suppliers", _("Недобросовестные поставщики ГОЗ")
|
||||
ARBITRATION = "arbitration", _("Арбитражные дела")
|
||||
SECURITY_REGISTRIES = "security_registries", _("Реестры по информационной безопасности")
|
||||
VACANCIES = "vacancies", _("Вакансии")
|
||||
|
||||
|
||||
class SourceExtensionStatus(models.TextChoices):
|
||||
"""Lifecycle status for an organization source extension."""
|
||||
|
||||
ACTIVE = "active", _("Активно")
|
||||
INACTIVE = "inactive", _("Неактивно")
|
||||
ERROR = "error", _("Ошибка")
|
||||
|
||||
|
||||
class Organization(models.Model):
|
||||
"""Canonical organization without source-specific relations."""
|
||||
|
||||
class IdentityStatus(models.TextChoices):
|
||||
"""Completeness of legal identity identifiers."""
|
||||
|
||||
COMPLETE = "complete", _("Полная")
|
||||
PARTIAL = "partial", _("Частичная")
|
||||
MISSING = "missing", _("Отсутствует")
|
||||
|
||||
uid = models.UUIDField(
|
||||
_("UID"),
|
||||
primary_key=True,
|
||||
@@ -53,6 +83,22 @@ class Organization(models.Model):
|
||||
db_index=True,
|
||||
help_text=_("ОГРИП только для индивидуальных предпринимателей"),
|
||||
)
|
||||
identity_status = models.CharField(
|
||||
_("полнота реквизитов"),
|
||||
max_length=16,
|
||||
choices=IdentityStatus.choices,
|
||||
default=IdentityStatus.MISSING,
|
||||
db_index=True,
|
||||
help_text=_("Оценка полноты идентификационных реквизитов"),
|
||||
)
|
||||
primary_identity = models.CharField(
|
||||
_("основной идентификатор"),
|
||||
max_length=255,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
editable=False,
|
||||
help_text=_("Нормализованный ключ для диагностики и дедупликации"),
|
||||
)
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_organization"
|
||||
@@ -96,6 +142,379 @@ class Organization(models.Model):
|
||||
def normalized_name(self) -> str:
|
||||
return normalize_organization_name(self.name)
|
||||
|
||||
def save(self, *args, **kwargs) -> None:
|
||||
self.identity_status = self._resolve_identity_status()
|
||||
self.primary_identity = self._resolve_primary_identity()
|
||||
update_fields = kwargs.get("update_fields")
|
||||
if update_fields is not None:
|
||||
kwargs["update_fields"] = list(
|
||||
dict.fromkeys([*update_fields, "identity_status", "primary_identity"])
|
||||
)
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
def _resolve_identity_status(self) -> str:
|
||||
if self.inn and (self.ogrn or self.ogrip):
|
||||
return self.IdentityStatus.COMPLETE
|
||||
if self.inn or self.ogrn or self.ogrip:
|
||||
return self.IdentityStatus.PARTIAL
|
||||
return self.IdentityStatus.MISSING
|
||||
|
||||
def _resolve_primary_identity(self) -> str:
|
||||
if self.inn and self.kpp:
|
||||
return f"inn:{self.inn}:kpp:{self.kpp}"
|
||||
if self.ogrn:
|
||||
return f"ogrn:{self.ogrn}"
|
||||
if self.ogrip:
|
||||
return f"ogrip:{self.ogrip}"
|
||||
if self.inn:
|
||||
return f"inn:{self.inn}"
|
||||
normalized_name = normalize_organization_name(self.name)
|
||||
if normalized_name:
|
||||
return f"name:{normalized_name}"[:255]
|
||||
return ""
|
||||
|
||||
|
||||
class OrganizationSourceExtension(PolymorphicModel):
|
||||
"""Base source group extension for one canonical organization."""
|
||||
|
||||
uid = models.UUIDField(
|
||||
_("UID"),
|
||||
primary_key=True,
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
)
|
||||
organization = models.ForeignKey(
|
||||
Organization,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="source_extensions",
|
||||
verbose_name=_("организация"),
|
||||
)
|
||||
source_group = models.CharField(
|
||||
_("группа источников"),
|
||||
max_length=64,
|
||||
choices=SourceGroup.choices,
|
||||
db_index=True,
|
||||
)
|
||||
title = models.CharField(
|
||||
_("название"),
|
||||
max_length=255,
|
||||
help_text=_("Человекочитаемое название блока источника"),
|
||||
)
|
||||
status = models.CharField(
|
||||
_("статус"),
|
||||
max_length=16,
|
||||
choices=SourceExtensionStatus.choices,
|
||||
default=SourceExtensionStatus.ACTIVE,
|
||||
db_index=True,
|
||||
)
|
||||
records_count = models.PositiveIntegerField(
|
||||
_("количество записей"),
|
||||
default=0,
|
||||
)
|
||||
first_seen_at = models.DateTimeField(
|
||||
_("первая запись"),
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
last_seen_at = models.DateTimeField(
|
||||
_("последняя запись"),
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
last_load_batch = models.PositiveIntegerField(
|
||||
_("последний пакет загрузки"),
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
metadata = models.JSONField(
|
||||
_("метаданные"),
|
||||
default=dict,
|
||||
blank=True,
|
||||
)
|
||||
created_at = models.DateTimeField(_("дата создания"), auto_now_add=True)
|
||||
updated_at = models.DateTimeField(_("дата обновления"), auto_now=True)
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_source_extension"
|
||||
verbose_name = _("расширение источника организации")
|
||||
verbose_name_plural = _("расширения источников организаций")
|
||||
ordering = ["organization__name", "source_group"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["organization", "source_group"],
|
||||
name="unique_organization_source_group_extension",
|
||||
),
|
||||
]
|
||||
indexes = [
|
||||
models.Index(fields=["source_group", "status"]),
|
||||
models.Index(fields=["organization", "source_group"]),
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.organization}: {self.title}"
|
||||
|
||||
def save(self, *args, **kwargs) -> None:
|
||||
source_group = getattr(self, "source_group_value", "")
|
||||
if source_group:
|
||||
self.source_group = source_group
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
|
||||
class FinancialIndicatorsExtension(OrganizationSourceExtension):
|
||||
"""Financial and accounting indicators source group."""
|
||||
|
||||
source_group_value = SourceGroup.FINANCIAL_INDICATORS
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_financial_indicators_extension"
|
||||
verbose_name = _("финансово-экономические показатели")
|
||||
verbose_name_plural = _("финансово-экономические показатели")
|
||||
|
||||
|
||||
class GovernmentProcurementExtension(OrganizationSourceExtension):
|
||||
"""Government procurement source group."""
|
||||
|
||||
source_group_value = SourceGroup.GOVERNMENT_PROCUREMENTS
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_government_procurement_extension"
|
||||
verbose_name = _("государственные закупки")
|
||||
verbose_name_plural = _("государственные закупки")
|
||||
|
||||
|
||||
class IndustrialProductionExtension(OrganizationSourceExtension):
|
||||
"""Russian manufacturers and products source group."""
|
||||
|
||||
source_group_value = SourceGroup.INDUSTRIAL_PRODUCTION
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_industrial_production_extension"
|
||||
verbose_name = _("производители и продукция России")
|
||||
verbose_name_plural = _("производители и продукция России")
|
||||
|
||||
|
||||
class PlannedInspectionExtension(OrganizationSourceExtension):
|
||||
"""Planned inspections source group."""
|
||||
|
||||
source_group_value = SourceGroup.PLANNED_INSPECTIONS
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_planned_inspection_extension"
|
||||
verbose_name = _("плановые проверки")
|
||||
verbose_name_plural = _("плановые проверки")
|
||||
|
||||
|
||||
class BankruptcyExtension(OrganizationSourceExtension):
|
||||
"""Bankruptcy procedures source group."""
|
||||
|
||||
source_group_value = SourceGroup.BANKRUPTCY
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_bankruptcy_extension"
|
||||
verbose_name = _("сведения о процедурах банкротства")
|
||||
verbose_name_plural = _("сведения о процедурах банкротства")
|
||||
|
||||
|
||||
class DefenseSupplierExtension(OrganizationSourceExtension):
|
||||
"""Defense supplier risk source group."""
|
||||
|
||||
source_group_value = SourceGroup.DEFENSE_SUPPLIERS
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_defense_supplier_extension"
|
||||
verbose_name = _("недобросовестные поставщики ГОЗ")
|
||||
verbose_name_plural = _("недобросовестные поставщики ГОЗ")
|
||||
|
||||
|
||||
class ArbitrationExtension(OrganizationSourceExtension):
|
||||
"""Arbitration cases source group."""
|
||||
|
||||
source_group_value = SourceGroup.ARBITRATION
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_arbitration_extension"
|
||||
verbose_name = _("арбитражные дела")
|
||||
verbose_name_plural = _("арбитражные дела")
|
||||
|
||||
|
||||
class SecurityRegistryExtension(OrganizationSourceExtension):
|
||||
"""Information security registries source group."""
|
||||
|
||||
source_group_value = SourceGroup.SECURITY_REGISTRIES
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_security_registry_extension"
|
||||
verbose_name = _("реестры по информационной безопасности")
|
||||
verbose_name_plural = _("реестры по информационной безопасности")
|
||||
|
||||
|
||||
class VacancyExtension(OrganizationSourceExtension):
|
||||
"""Vacancies source group."""
|
||||
|
||||
source_group_value = SourceGroup.VACANCIES
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_vacancy_extension"
|
||||
verbose_name = _("вакансии")
|
||||
verbose_name_plural = _("вакансии")
|
||||
|
||||
|
||||
class OrganizationSourceRecord(models.Model):
|
||||
"""Subordinate source record stored under a source extension."""
|
||||
|
||||
uid = models.UUIDField(
|
||||
_("UID"),
|
||||
primary_key=True,
|
||||
default=uuid.uuid4,
|
||||
editable=False,
|
||||
)
|
||||
extension = models.ForeignKey(
|
||||
OrganizationSourceExtension,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="records",
|
||||
verbose_name=_("расширение источника"),
|
||||
)
|
||||
record_type = models.CharField(
|
||||
_("тип записи"),
|
||||
max_length=64,
|
||||
db_index=True,
|
||||
)
|
||||
source = models.CharField(
|
||||
_("источник"),
|
||||
max_length=64,
|
||||
db_index=True,
|
||||
)
|
||||
external_id = models.CharField(
|
||||
_("внешний ID"),
|
||||
max_length=255,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
title = models.TextField(
|
||||
_("заголовок"),
|
||||
blank=True,
|
||||
)
|
||||
record_date = models.CharField(
|
||||
_("дата записи"),
|
||||
max_length=255,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
amount = models.DecimalField(
|
||||
_("сумма"),
|
||||
max_digits=20,
|
||||
decimal_places=2,
|
||||
null=True,
|
||||
blank=True,
|
||||
)
|
||||
status = models.CharField(
|
||||
_("статус"),
|
||||
max_length=255,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
url = models.TextField(
|
||||
_("URL"),
|
||||
blank=True,
|
||||
)
|
||||
payload = models.JSONField(
|
||||
_("исходные данные"),
|
||||
default=dict,
|
||||
blank=True,
|
||||
)
|
||||
legacy_model = models.CharField(
|
||||
_("legacy model"),
|
||||
max_length=255,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
legacy_pk = models.CharField(
|
||||
_("legacy pk"),
|
||||
max_length=64,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
load_batch = models.PositiveIntegerField(
|
||||
_("ID пакета загрузки"),
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
created_at = models.DateTimeField(_("дата создания"), auto_now_add=True)
|
||||
updated_at = models.DateTimeField(_("дата обновления"), auto_now=True)
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_source_record"
|
||||
verbose_name = _("запись источника организации")
|
||||
verbose_name_plural = _("записи источников организаций")
|
||||
ordering = ["-created_at"]
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["source", "external_id"],
|
||||
condition=~Q(external_id=""),
|
||||
name="unique_source_record_external_id",
|
||||
),
|
||||
models.UniqueConstraint(
|
||||
fields=["legacy_model", "legacy_pk"],
|
||||
condition=~Q(legacy_model="") & ~Q(legacy_pk=""),
|
||||
name="unique_source_record_legacy_identity",
|
||||
),
|
||||
]
|
||||
indexes = [
|
||||
models.Index(fields=["extension", "source"]),
|
||||
models.Index(fields=["source", "record_type"]),
|
||||
models.Index(fields=["load_batch", "source"]),
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.title or self.external_id or str(self.uid)
|
||||
|
||||
|
||||
class OrganizationSourceFinancialLine(models.Model):
|
||||
"""Structured financial report line under a source record."""
|
||||
|
||||
source_record = models.ForeignKey(
|
||||
OrganizationSourceRecord,
|
||||
on_delete=models.CASCADE,
|
||||
related_name="financial_lines",
|
||||
verbose_name=_("запись источника"),
|
||||
)
|
||||
form_code = models.CharField(_("код формы"), max_length=10, db_index=True)
|
||||
line_code = models.CharField(_("код строки"), max_length=10, db_index=True)
|
||||
line_name = models.CharField(_("наименование строки"), max_length=255)
|
||||
year = models.PositiveSmallIntegerField(_("год"), db_index=True)
|
||||
period_start = models.BigIntegerField(
|
||||
_("на начало периода"),
|
||||
null=True,
|
||||
blank=True,
|
||||
)
|
||||
period_end = models.BigIntegerField(
|
||||
_("на конец периода"),
|
||||
null=True,
|
||||
blank=True,
|
||||
)
|
||||
|
||||
class Meta:
|
||||
db_table = "organizations_source_financial_line"
|
||||
verbose_name = _("строка финансового источника")
|
||||
verbose_name_plural = _("строки финансовых источников")
|
||||
constraints = [
|
||||
models.UniqueConstraint(
|
||||
fields=["source_record", "form_code", "line_code", "year"],
|
||||
name="unique_source_financial_line_year",
|
||||
),
|
||||
]
|
||||
indexes = [
|
||||
models.Index(fields=["source_record", "form_code", "line_code"]),
|
||||
models.Index(fields=["year", "line_code"]),
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.line_code} ({self.line_name[:30]}) - {self.year}"
|
||||
|
||||
|
||||
class OrganizationDataSnapshot(models.Model):
|
||||
"""Precomputed API v2 data payload for one canonical organization."""
|
||||
|
||||
Reference in New Issue
Block a user