avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(organizations): migrate source storage to polymorphic records

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-05-19 10:23:53 +02:00
parent 19a7d5a91c
commit 4ca2fa25d5
44 changed files with 7129 additions and 1551 deletions
Download Patch File Download Diff File Expand all files Collapse all files

786
src/apps/parsers/services.py
View File

File diff suppressed because it is too large Load Diff

232
src/apps/parsers/source_cards.py
View File

@@ -12,21 +12,13 @@ from apps.core.models import JobStatus
from apps.core.services import BackgroundJobService
from apps.parsers.models import (
VACANCY_RECORD_SOURCES,
FinancialReport,
FinancialReportLine,
GenericParserRecord,
IndustrialCertificateRecord,
IndustrialProductRecord,
InspectionRecord,
ManufacturerRecord,
ParserLoadLog,
ProcurementRecord,
)
from django.conf import settings
from django.db.models import CharField, F, Max, Q, Value
from django.db.models.functions import Coalesce, NullIf
from django.db.models import Max, Q
from django.http import Http404
from django.utils import timezone
from organizations.models import OrganizationSourceRecord
from rest_framework.exceptions import ValidationError
SUCCESSFUL_LOAD_STATUSES = {"success", "skipped"}
@@ -328,10 +320,16 @@ GENERIC_RECORD_SOURCES_BY_ITEM_CODE = {
"fstec": ParserLoadLog.Source.FSTEC,
"trudvsem": ParserLoadLog.Source.TRUDVSEM,
}
PROCUREMENT_BUYER_ITEM_CODES = {
"procurements_44fz",
"procurements_223fz",
"contracts",
SOURCE_RECORD_SOURCES_BY_ITEM_CODE = {
item.code: (
list(VACANCY_RECORD_SOURCES)
if item.parser_source == ParserLoadLog.Source.TRUDVSEM
else [item.parser_source]
)
for definition in SOURCE_CARD_DEFINITIONS
for item in definition.source_items
if item.parser_source
}
@@ -779,116 +777,22 @@ class SourceCardService:
@classmethod
def _get_source_records_count(cls, item_code: str) -> int:
generic_sources = cls._get_generic_sources_for_item_code(item_code)
if generic_sources:
return GenericParserRecord.objects.filter(
source__in=generic_sources
).count()
if item_code == "fns_reports":
return FinancialReportLine.objects.count()
if item_code == "industrial":
return IndustrialCertificateRecord.objects.count()
if item_code == "manufactures":
return ManufacturerRecord.objects.count()
if item_code == "industrial_products":
return IndustrialProductRecord.objects.count()
if item_code == "inspections":
return InspectionRecord.objects.count()
if item_code == "procurements":
return ProcurementRecord.objects.count()
return 0
return cls._get_source_record_queryset(item_code).count()
@classmethod
def _get_source_organizations_count(cls, item_code: str) -> int:
generic_sources = cls._get_generic_sources_for_item_code(item_code)
if generic_sources:
if item_code in PROCUREMENT_BUYER_ITEM_CODES:
return cls._get_generic_procurement_buyer_identities(
generic_sources
).count()
return (
GenericParserRecord.objects.filter(source__in=generic_sources)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if item_code == "fns_reports":
return (
FinancialReport.objects.exclude(ogrn="")
.values("ogrn")
.distinct()
.count()
)
if item_code == "industrial":
return (
IndustrialCertificateRecord.objects.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if item_code == "manufactures":
return (
ManufacturerRecord.objects.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if item_code == "industrial_products":
return (
IndustrialProductRecord.objects.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if item_code == "inspections":
return (
InspectionRecord.objects.exclude(inn="")
.values("inn")
.distinct()
.count()
)
if item_code == "procurements":
return (
ProcurementRecord.objects.exclude(customer_inn="")
.values("customer_inn")
.distinct()
.count()
)
return 0
return (
cls._get_source_record_queryset(item_code)
.values("extension__organization_id")
.distinct()
.count()
)
@classmethod
def _get_source_data_timestamp(cls, item_code: str):
generic_sources = cls._get_generic_sources_for_item_code(item_code)
if generic_sources:
return GenericParserRecord.objects.filter(
source__in=generic_sources
).aggregate(last_updated=Max("updated_at"))["last_updated"]
if item_code == "fns_reports":
return FinancialReport.objects.aggregate(last_updated=Max("updated_at"))[
"last_updated"
]
if item_code == "industrial":
return IndustrialCertificateRecord.objects.aggregate(
last_updated=Max("updated_at")
)["last_updated"]
if item_code == "manufactures":
return ManufacturerRecord.objects.aggregate(last_updated=Max("updated_at"))[
"last_updated"
]
if item_code == "industrial_products":
return IndustrialProductRecord.objects.aggregate(
last_updated=Max("updated_at")
)["last_updated"]
if item_code == "inspections":
return InspectionRecord.objects.aggregate(last_updated=Max("updated_at"))[
"last_updated"
]
if item_code == "procurements":
return ProcurementRecord.objects.aggregate(last_updated=Max("updated_at"))[
"last_updated"
]
return None
return cls._get_source_record_queryset(item_code).aggregate(
last_updated=Max("updated_at")
)["last_updated"]
@classmethod
def _get_card_organizations_count(
@@ -896,54 +800,32 @@ class SourceCardService:
definition: SourceCardDefinition,
source_items: list[dict[str, Any]],
) -> int:
if definition.slug == "public-procurements":
generic_sources = cls._get_generic_sources_for_definition(definition)
legacy_inns = (
ProcurementRecord.objects.exclude(customer_inn="")
.annotate(buyer_identity=F("customer_inn"))
.order_by()
.values_list("buyer_identity", flat=True)
.distinct()
)
generic_inns = cls._get_generic_procurement_buyer_identities(
generic_sources
)
return legacy_inns.union(generic_inns).count()
if definition.slug != "manufacturers-and-products":
generic_sources = cls._get_generic_sources_for_definition(definition)
if generic_sources and len(generic_sources) == len(definition.source_items):
return (
GenericParserRecord.objects.filter(source__in=generic_sources)
.exclude(inn="")
.values("inn")
.distinct()
.count()
)
source_codes = [item.code for item in definition.source_items]
sources = cls._get_sources_for_item_codes(source_codes)
if not sources:
return sum(item["organizations_count"] for item in source_items)
industrial_inns = (
IndustrialCertificateRecord.objects.exclude(inn="")
.order_by()
.values_list("inn", flat=True)
return (
OrganizationSourceRecord.objects.filter(source__in=sources)
.values("extension__organization_id")
.distinct()
.count()
)
manufacturer_inns = (
ManufacturerRecord.objects.exclude(inn="")
.order_by()
.values_list("inn", flat=True)
.distinct()
)
product_inns = (
IndustrialProductRecord.objects.exclude(inn="")
.order_by()
.values_list("inn", flat=True)
.distinct()
)
return industrial_inns.union(manufacturer_inns, product_inns).count()
@staticmethod
def _get_generic_sources_for_item_code(item_code: str) -> list[str]:
return SourceCardService._get_sources_for_item_code(item_code)
@staticmethod
def _get_source_record_queryset(item_code: str):
return OrganizationSourceRecord.objects.filter(
source__in=SourceCardService._get_sources_for_item_code(item_code),
)
@staticmethod
def _get_sources_for_item_code(item_code: str) -> list[str]:
sources = SOURCE_RECORD_SOURCES_BY_ITEM_CODE.get(item_code)
if sources:
return list(sources)
generic_source = GENERIC_RECORD_SOURCES_BY_ITEM_CODE.get(item_code)
if not generic_source:
return []
@@ -952,35 +834,21 @@ class SourceCardService:
return [generic_source]
@staticmethod
def _get_generic_procurement_buyer_identities(generic_sources: list[str]):
return (
GenericParserRecord.objects.filter(source__in=generic_sources)
.annotate(
buyer_identity=Coalesce(
NullIf(F("inn"), Value("")),
NullIf(F("organisation_name"), Value("")),
output_field=CharField(),
)
def _get_sources_for_item_codes(item_codes: list[str]) -> list[str]:
return list(
dict.fromkeys(
source
for item_code in item_codes
for source in SourceCardService._get_sources_for_item_code(item_code)
)
.exclude(buyer_identity__isnull=True)
.order_by()
.values_list("buyer_identity", flat=True)
.distinct()
)
@staticmethod
def _get_generic_sources_for_definition(
definition: SourceCardDefinition,
) -> list[str]:
return list(
dict.fromkeys(
source
for item in definition.source_items
if item.code in GENERIC_RECORD_SOURCES_BY_ITEM_CODE
for source in SourceCardService._get_generic_sources_for_item_code(
item.code
)
)
return SourceCardService._get_sources_for_item_codes(
[item.code for item in definition.source_items]
)
@classmethod

84
src/apps/parsers/tasks.py
View File

@@ -54,7 +54,6 @@ from apps.parsers.services import (
from apps.parsers.source_registry import PARSER_SOURCES
from celery import shared_task
from django.conf import settings
from django.db import transaction
from registers.models import RegistryMembershipPeriod
from requests.adapters import BaseAdapter
@@ -226,22 +225,6 @@ def _get_or_create_background_job(
return job
def _queue_organization_snapshot_refresh(source: str, batch_id: int) -> None:
"""Queue snapshot refresh only after parser writes are committed."""
def enqueue() -> None:
from organizations.tasks import (
refresh_organization_data_snapshots_for_parser_batch,
)
refresh_organization_data_snapshots_for_parser_batch.delay(
source=str(source),
batch_id=batch_id,
)
transaction.on_commit(enqueue)
def _run_generic_parser(
self,
*,
@@ -284,7 +267,6 @@ def _run_generic_parser(
)
result = {"batch_id": batch_id, "saved": saved_count, "status": "success"}
job.complete(result=result)
_queue_organization_snapshot_refresh(source, batch_id)
return result
except ParserSourceSkipped as e:
message = str(e)
@@ -350,7 +332,6 @@ def _run_inspection_parser(
)
result = {"batch_id": batch_id, "saved": saved_count, "status": "success"}
job.complete(result=result)
_queue_organization_snapshot_refresh(source, batch_id)
return result
except ParserSourceSkipped as e:
message = str(e)
@@ -523,20 +504,36 @@ def _checko_bankruptcy_items(
fallback_name: str,
) -> list[GenericParserItem]:
"""Преобразовать банкротные сообщения Checko в generic records."""
records: list[GenericParserItem] = []
inn = str(getattr(company, "inn", "") or fallback_inn)
ogrn = str(getattr(company, "ogrn", "") or fallback_ogrn)
name = getattr(company, "short_name", None) or fallback_name
messages_by_external_id: dict[str, list[dict[str, str]]] = {}
for message in getattr(company, "bankruptcy", ()):
message_type = getattr(message, "type", "") or "Сообщение ЕФРСБ"
message_date = getattr(message, "date", "") or ""
case_number = getattr(message, "case_number", None) or ""
external_id = _fedresurs_external_id(
inn=inn,
message_type=message_type,
message_date=message_date,
ogrn=ogrn,
case_number=case_number,
)
messages_by_external_id.setdefault(external_id, []).append(
{
"case_number": case_number,
"date": message_date,
"type": message_type,
}
)
records: list[GenericParserItem] = []
for external_id, messages in messages_by_external_id.items():
messages = sorted(
messages,
key=lambda item: item["date"],
reverse=True,
)
latest_message = messages[0]
records.append(
GenericParserItem(
source="fedresurs_bankruptcy",
@@ -544,18 +541,20 @@ def _checko_bankruptcy_items(
inn=inn,
ogrn=ogrn,
organisation_name=name,
title=message_type,
record_date=message_date,
status=message_type,
title=latest_message["type"],
record_date=latest_message["date"],
status=latest_message["type"],
payload={
"provider": "checko",
"declared_source": "ЕФРСБ",
"inn": inn,
"ogrn": ogrn,
"organisation_name": name,
"type": message_type,
"date": message_date,
"case_number": case_number,
"type": latest_message["type"],
"date": latest_message["date"],
"case_number": latest_message["case_number"],
"messages": messages,
"messages_count": len(messages),
},
)
)
@@ -565,14 +564,15 @@ def _checko_bankruptcy_items(
def _fedresurs_external_id(
*,
inn: str,
message_type: str,
message_date: str,
case_number: str,
ogrn: str = "",
) -> str:
"""Стабильный ID для ЕФРСБ-сообщения из fallback-источника."""
raw = f"{inn}:{message_type}:{message_date}:{case_number}"
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:24]
return f"checko-fedresurs:{digest}"
"""Стабильный ID для процедуры банкротства из fallback-источника."""
identity = _normalize_identifier(inn) or _normalize_identifier(ogrn)
normalized_case_number = re.sub(r"\s+", "", str(case_number or "")).upper()
if normalized_case_number:
return f"checko-fedresurs:{identity}:{normalized_case_number}"
return f"checko-fedresurs:{identity}"
@dataclass(frozen=True)
@@ -1272,24 +1272,23 @@ def _process_fns_file_sync(
# Завершаем
job.complete(
result={
"report_id": report.id,
"source_record_uid": str(report.uid),
"external_id": parsed.external_id,
"ogrn": parsed.ogrn,
"lines_count": len(parsed.lines),
}
)
_queue_organization_snapshot_refresh(source, batch_id)
logger.info(
"FNS file processed: %s (report_id=%d, lines=%d)",
"FNS file processed: %s (source_record_uid=%s, lines=%d)",
file_path.name,
report.id,
report.uid,
len(parsed.lines),
)
return {
"status": "success",
"report_id": report.id,
"source_record_uid": str(report.uid),
"external_id": parsed.external_id,
"ogrn": parsed.ogrn,
"lines_count": len(parsed.lines),
@@ -1399,7 +1398,6 @@ def parse_industrial_production(
# Завершаем BackgroundJob
job.complete(result={"batch_id": batch_id, "saved": saved_count})
_queue_organization_snapshot_refresh(source, batch_id)
logger.info(
"Industrial production parsing completed (batch_id=%d, saved=%d)",
@@ -1492,7 +1490,6 @@ def parse_manufactures(
# Завершаем BackgroundJob
job.complete(result={"batch_id": batch_id, "saved": saved_count})
_queue_organization_snapshot_refresh(source, batch_id)
logger.info(
"Manufactures parsing completed (batch_id=%d, saved=%d)",
@@ -1581,7 +1578,6 @@ def parse_industrial_products(
)
job.complete(result={"batch_id": batch_id, "saved": saved_count})
_queue_organization_snapshot_refresh(source, batch_id)
logger.info(
"Industrial products parsing completed (batch_id=%d, saved=%d)",
@@ -1749,7 +1745,6 @@ def parse_inspections(
# Завершаем BackgroundJob
job.complete(result={"batch_id": batch_id, "saved": saved_count})
_queue_organization_snapshot_refresh(source, batch_id)
logger.info(
"Inspections parsing completed (batch_id=%d, saved=%d)",
@@ -2124,7 +2119,6 @@ def sync_inspections( # noqa: C901
"results": results,
}
)
_queue_organization_snapshot_refresh(source, batch_id)
logger.info("Inspections sync completed (total_saved=%d)", total_saved)
@@ -2257,7 +2251,6 @@ def parse_procurements(
# Завершаем BackgroundJob
job.complete(result={"batch_id": batch_id, "saved": saved_count})
_queue_organization_snapshot_refresh(source, batch_id)
logger.info(
"Procurements parsing completed (batch_id=%d, saved=%d)",
@@ -2474,7 +2467,6 @@ def sync_procurements( # noqa: C901
"results": results,
}
)
_queue_organization_snapshot_refresh(source, batch_id)
logger.info("Procurements sync completed (total_saved=%d)", total_saved)

86
src/apps/parsers/views.py
View File

@@ -78,6 +78,7 @@ from django_celery_beat.models import CrontabSchedule, IntervalSchedule, Periodi
from drf_yasg import openapi
from drf_yasg.inspectors import SwaggerAutoSchema
from drf_yasg.utils import no_body, swagger_auto_schema
from organizations.models import OrganizationSourceRecord
from rest_framework import status
from rest_framework.exceptions import ValidationError
from rest_framework.parsers import FormParser, JSONParser, MultiPartParser
@@ -1757,28 +1758,47 @@ def _matched_registry_organization_ids(
return matched_ids
def _source_record_queryset_for_parser_source(source: str):
if source == ParserLoadLog.Source.TRUDVSEM:
return OrganizationSourceRecord.objects.filter(source__in=VACANCY_RECORD_SOURCES)
return OrganizationSourceRecord.objects.filter(source=source)
def _matched_source_record_registry_organization_ids(
source: str,
by_inn: dict[str, set[int]],
by_ogrn: dict[str, set[int]],
) -> set[int]:
matched_ids: set[int] = set()
queryset = _source_record_queryset_for_parser_source(source)
for inn, ogrn, ogrip in (
queryset.order_by()
.values_list(
"extension__organization__inn",
"extension__organization__ogrn",
"extension__organization__ogrip",
)
.distinct()
):
inn_text = _normalize_registry_identifier(inn)
ogrn_text = _normalize_registry_identifier(ogrn or ogrip)
if inn_text:
matched_ids.update(by_inn.get(inn_text, ()))
if ogrn_text:
matched_ids.update(by_ogrn.get(ogrn_text, ()))
return matched_ids
def _source_registry_matches(
by_inn: dict[str, set[int]],
by_ogrn: dict[str, set[int]],
) -> dict[str, set[int]]:
matches: dict[str, set[int]] = {}
for source, _label in ParserLoadLog.Source.choices:
inn_field, ogrn_field = REGISTRY_COVERAGE_NATIVE_IDENTITY_FIELDS.get(
matches[source] = _matched_source_record_registry_organization_ids(
source,
("inn", "ogrn"),
)
if source == ParserLoadLog.Source.FNS_REPORTS:
queryset = FinancialReport.objects.all()
elif source in NATIVE_RECORD_MODELS:
queryset = NATIVE_RECORD_MODELS[source].objects.all()
else:
queryset = GenericParserRecord.objects.filter(source=source)
matches[source] = _matched_registry_organization_ids(
queryset,
by_inn,
by_ogrn,
inn_field=inn_field,
ogrn_field=ogrn_field,
)
return matches
@@ -1861,32 +1881,9 @@ def _registry_data_coverage() -> dict:
for source, _label in ParserLoadLog.Source.choices:
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
continue
inn_field, ogrn_field = REGISTRY_COVERAGE_NATIVE_IDENTITY_FIELDS.get(
source,
("inn", "ogrn"),
organizations_count = len(
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
)
if source == ParserLoadLog.Source.FNS_REPORTS:
organizations_count = _matched_registry_organization_count(
FinancialReport.objects.all(),
by_inn,
by_ogrn,
inn_field=inn_field,
ogrn_field=ogrn_field,
)
elif source in NATIVE_RECORD_MODELS:
organizations_count = _matched_registry_organization_count(
NATIVE_RECORD_MODELS[source].objects.all(),
by_inn,
by_ogrn,
inn_field=inn_field,
ogrn_field=ogrn_field,
)
else:
organizations_count = _matched_registry_organization_count(
GenericParserRecord.objects.filter(source=source),
by_inn,
by_ogrn,
)
coverage_percent = (
round(organizations_count / total_organizations * 100, 1)
if total_organizations
@@ -2781,20 +2778,11 @@ class ParserDashboardDataView(APIView):
file_sources = [source for source in sources if source["supports_file_upload"]]
jobs = BackgroundJobService.get_user_jobs(user_id=request.user.id, limit=30)
source_counts = dict(
GenericParserRecord.objects.order_by()
OrganizationSourceRecord.objects.order_by()
.values("source")
.annotate(count=Count("id"))
.annotate(count=Count("uid"))
.values_list("source", "count")
)
source_counts.update(
{
source: model.objects.count()
for source, model in NATIVE_RECORD_MODELS.items()
}
)
source_counts.update(
{ParserLoadLog.Source.FNS_REPORTS: FinancialReport.objects.count()}
)
schedules = [
_periodic_task_to_dict(task)
for task in _parser_periodic_tasks_for_user(request.user)