feat: add Fedresurs profiles and integrated structures sources
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 6m35s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped

This commit is contained in:
Aleksandr Meshchryakov
2026-09-24 14:14:01 +02:00
parent 4c869b856c
commit 85a3c0650e
42 changed files with 3166 additions and 49 deletions

View File

@@ -0,0 +1,306 @@
"""Atomic XLSX/RTF import for integrated structures."""
from __future__ import annotations
import hashlib
import re
from collections.abc import Iterable
from dataclasses import dataclass
from io import BytesIO
from typing import BinaryIO
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
ParsedWorkbook,
parse_integrated_workbook,
)
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
IntegratedStructureEvent,
IntegratedStructureEventDocument,
ParserLoadLog,
ParserSourceArtifact,
ParserStagedRecord,
)
from apps.parsers.registry_snapshots import (
publish_snapshot,
reject_snapshot,
save_artifact_file,
stable_registry_uid,
stage_records,
)
from apps.parsers.services import ParserLoadLogService
from django.core.files.base import ContentFile
from organizations.models import Organization, OrganizationSourceRecord
PARSER_SOURCE = "integrated_structure_documents"
RECORD_SOURCE = "integrated_structures"
RECORD_TYPE = "integrated_structure"
RTF_NAME = re.compile(r"^([0-9]{1,12})\.rtf$", re.IGNORECASE)
MAX_ATTACHMENTS = 500
MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024
MAX_TOTAL_ATTACHMENT_BYTES = 100 * 1024 * 1024
@dataclass(frozen=True)
class ValidatedAttachment:
document_id: str
name: str
sha256: str
data: bytes
def _resolve_attachment_document_id(raw_id: str, parsed: ParsedWorkbook) -> str:
if raw_id in parsed.documents:
return raw_id
candidates = [
candidate for candidate in parsed.documents if int(candidate) == int(raw_id)
]
if len(candidates) != 1:
raise IntegratedWorkbookError("unknown_or_ambiguous_attachment_document")
return candidates[0]
def _validated_attachments(
files: Iterable[BinaryIO], parsed: ParsedWorkbook
) -> dict[str, ValidatedAttachment]:
result: dict[str, ValidatedAttachment] = {}
total_size = 0
for count, file in enumerate(files, 1):
if count > MAX_ATTACHMENTS:
raise IntegratedWorkbookError("too_many_attachments")
name = str(getattr(file, "name", ""))
match = RTF_NAME.fullmatch(name)
if match is None:
raise IntegratedWorkbookError("invalid_attachment_name")
document_id = _resolve_attachment_document_id(match[1], parsed)
if document_id in result:
raise IntegratedWorkbookError("duplicate_attachment")
if document_id not in parsed.documents:
raise IntegratedWorkbookError("unknown_attachment_document")
file.seek(0)
data = file.read(MAX_ATTACHMENT_BYTES + 1)
if len(data) > MAX_ATTACHMENT_BYTES:
raise IntegratedWorkbookError("attachment_too_large")
total_size += len(data)
if total_size > MAX_TOTAL_ATTACHMENT_BYTES:
raise IntegratedWorkbookError("attachments_too_large")
if not data.startswith(b"{\\rtf"):
raise IntegratedWorkbookError("invalid_rtf_signature")
result[document_id] = ValidatedAttachment(
document_id=document_id,
name=name,
sha256=hashlib.sha256(data).hexdigest(),
data=data,
)
return result
def _composite_hash(
workbook: bytes, attachments: dict[str, ValidatedAttachment]
) -> str:
digest = hashlib.sha256()
digest.update(len(workbook).to_bytes(8, "big"))
digest.update(workbook)
for document_id, attachment in sorted(attachments.items()):
digest.update(document_id.encode("ascii"))
digest.update(attachment.sha256.encode("ascii"))
return digest.hexdigest()
def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
by_okpo: dict[str, Organization] = {}
for organization in Organization.objects.filter(
okpo__in=parsed.structures, directory_imported_at__isnull=False
).only("uid", "name", "inn", "ogrn", "kpp", "okpo"):
if organization.okpo in by_okpo:
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
by_okpo[organization.okpo] = organization
if len(by_okpo) != len(parsed.structures):
raise IntegratedWorkbookError("unknown_canonical_okpo")
return by_okpo
def _save_attachments(
attachments: dict[str, ValidatedAttachment],
) -> dict[str, IntegratedStructureAttachment]:
result = {}
for document_id, attachment in attachments.items():
stored = IntegratedStructureAttachment.objects.filter(
sha256=attachment.sha256
).first()
if stored is None:
stored = IntegratedStructureAttachment(
sha256=attachment.sha256, size_bytes=len(attachment.data)
)
stored.file.save(attachment.name, ContentFile(attachment.data), save=False)
stored.save()
result[document_id] = stored
return result
def _publish_relations(
artifact: ParserSourceArtifact,
parsed: ParsedWorkbook,
attachments: dict[str, ValidatedAttachment],
) -> None:
stored_attachments = _save_attachments(attachments)
documents = {}
for document_id, document in parsed.documents.items():
documents[document_id] = IntegratedStructureDocument.objects.create(
artifact=artifact,
external_id=document_id,
name=document.name,
document_type=document.document_type,
number=document.number,
document_date=document.date,
dsp=document.dsp,
have_text=document.have_text,
consultant_web_url=document.consultant_web_url or "",
attachment_filename=attachments[document_id].name
if document_id in attachments
else "",
attachment=stored_attachments.get(document_id),
)
for okpo, structure in parsed.structures.items():
external_id = f"integrated-structure:{okpo}"
record = OrganizationSourceRecord.objects.get(
uid=stable_registry_uid(RECORD_SOURCE, external_id)
)
record.integrated_events.all().delete()
for item in structure.events:
event = IntegratedStructureEvent.objects.create(
record=record,
external_id=item.external_id,
event_date=item.date,
note=item.note or "",
)
IntegratedStructureEventDocument.objects.bulk_create(
[
IntegratedStructureEventDocument(
event=event, document=documents[document_id]
)
for document_id in item.documents
]
)
def import_integrated_structures(
*,
workbook: BinaryIO,
original_name: str,
attachments: Iterable[BinaryIO],
uploaded_by_id: int | None = None,
) -> tuple[ParserSourceArtifact, dict]:
"""Validate all inputs, stage all structures, then publish one transaction."""
workbook.seek(0)
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
validated_attachments = _validated_attachments(attachments, parsed)
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
existing = (
ParserSourceArtifact.objects.filter(
source=PARSER_SOURCE,
status=ParserSourceArtifact.Status.PUBLISHED,
metadata__composite_sha256=composite_hash,
)
.order_by("-created_at")
.first()
)
if existing is not None:
return existing, {
"status": "skipped",
"reason": "identical_batch",
"batch_id": existing.load_batch,
"records_count": existing.published_count,
**existing.metadata,
}
organizations = _resolve_organizations(parsed)
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
)
artifact = ParserSourceArtifact.objects.create(
source=PARSER_SOURCE,
original_name=original_name[:512],
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
load_batch=batch_id,
uploaded_by_id=uploaded_by_id,
version="is_stages+is_stages_docs/v1",
parsed_count=len(parsed.structures),
metadata={
"composite_sha256": composite_hash,
"structures_count": parsed.counts[0],
"event_rows_count": parsed.counts[1],
"document_link_rows_count": parsed.counts[2],
"unique_documents_count": parsed.counts[3],
"events_without_documents_count": parsed.counts[4],
"attachments_count": len(validated_attachments),
"missing_attachments_count": len(parsed.documents)
- len(validated_attachments),
},
)
try:
save_artifact_file(artifact, BytesIO(workbook_bytes))
pending = []
for index, (okpo, structure) in enumerate(parsed.structures.items(), 1):
organization = organizations[okpo]
external_id = f"integrated-structure:{okpo}"
document_ids = {
doc_id for event in structure.events for doc_id in event.documents
}
event_dates = [event.date for event in structure.events if event.date]
pending.append(
ParserStagedRecord(
artifact=artifact,
row_number=index,
external_id=external_id,
record_type=RECORD_TYPE,
organization=organization,
normalized_data={
"title": organization.name,
"record_date": max(event_dates) if event_dates else "",
"status": "active",
"payload": {
"structure": {
"events_count": len(structure.events),
"document_links_count": sum(
len(event.documents) for event in structure.events
),
"unique_documents_count": len(document_ids),
"first_event_date": min(event_dates) if event_dates else "",
"latest_event_date": max(event_dates) if event_dates else "",
},
},
},
)
)
stage_records(pending)
artifact.status = ParserSourceArtifact.Status.PARSED
artifact.save(update_fields=["status", "updated_at"])
def finalize(artifact, result):
_publish_relations(artifact, parsed, validated_attachments)
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.SUCCESS,
records_count=result.published,
)
result = publish_snapshot(
artifact, record_source=RECORD_SOURCE, on_publish=finalize
)
return artifact, {
"status": "published",
"batch_id": batch_id,
"records_count": result.published,
**artifact.metadata,
}
except Exception as exc:
reject_snapshot(artifact, exc)
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.FAILED,
error_message=type(exc).__name__,
)
raise