Files
mostovik-backend/src/apps/parsers/integrated_structures.py
2026-09-28 11:27:46 +02:00

386 lines
14 KiB
Python

"""Atomic XLSX/RTF import for integrated structures."""
from __future__ import annotations
import hashlib
import re
from collections.abc import Iterable
from dataclasses import dataclass
from io import BytesIO
from typing import BinaryIO
from apps.parsers.integrated_structure_workbook import (
IntegratedWorkbookError,
ParsedWorkbook,
parse_integrated_workbook,
)
from apps.parsers.models import (
IntegratedStructureAttachment,
IntegratedStructureDocument,
IntegratedStructureEvent,
IntegratedStructureEventDocument,
ParserLoadLog,
ParserSourceArtifact,
ParserStagedRecord,
)
from apps.parsers.registry_snapshots import (
publish_snapshot,
reject_snapshot,
save_artifact_file,
stable_registry_uid,
stage_records,
)
from apps.parsers.services import ParserLoadLogService
from django.core.files.base import ContentFile
from organizations.models import Organization, OrganizationSourceRecord
PARSER_SOURCE = "integrated_structure_documents"
RECORD_SOURCE = "integrated_structures"
RECORD_TYPE = "integrated_structure"
RTF_NAME = re.compile(r"^([0-9]{1,12})\.rtf$", re.IGNORECASE)
MAX_ATTACHMENTS = 500
MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024
MAX_TOTAL_ATTACHMENT_BYTES = 100 * 1024 * 1024
@dataclass(frozen=True)
class ValidatedAttachment:
document_id: str
name: str
sha256: str
data: bytes
def _resolve_attachment_document_id(raw_id: str, parsed: ParsedWorkbook) -> str:
if raw_id in parsed.documents:
return raw_id
candidates = [
candidate for candidate in parsed.documents if int(candidate) == int(raw_id)
]
if len(candidates) != 1:
raise IntegratedWorkbookError("unknown_or_ambiguous_attachment_document")
return candidates[0]
def _validated_attachments(
files: Iterable[BinaryIO], parsed: ParsedWorkbook
) -> dict[str, ValidatedAttachment]:
result: dict[str, ValidatedAttachment] = {}
total_size = 0
for count, file in enumerate(files, 1):
if count > MAX_ATTACHMENTS:
raise IntegratedWorkbookError("too_many_attachments")
name = str(getattr(file, "name", ""))
match = RTF_NAME.fullmatch(name)
if match is None:
raise IntegratedWorkbookError("invalid_attachment_name")
document_id = _resolve_attachment_document_id(match[1], parsed)
if document_id in result:
raise IntegratedWorkbookError("duplicate_attachment")
if document_id not in parsed.documents:
raise IntegratedWorkbookError("unknown_attachment_document")
file.seek(0)
data = file.read(MAX_ATTACHMENT_BYTES + 1)
if len(data) > MAX_ATTACHMENT_BYTES:
raise IntegratedWorkbookError("attachment_too_large")
total_size += len(data)
if total_size > MAX_TOTAL_ATTACHMENT_BYTES:
raise IntegratedWorkbookError("attachments_too_large")
if not data.startswith(b"{\\rtf"):
raise IntegratedWorkbookError("invalid_rtf_signature")
result[document_id] = ValidatedAttachment(
document_id=document_id,
name=name,
sha256=hashlib.sha256(data).hexdigest(),
data=data,
)
return result
def _composite_hash(
workbook: bytes, attachments: dict[str, ValidatedAttachment]
) -> str:
digest = hashlib.sha256()
digest.update(len(workbook).to_bytes(8, "big"))
digest.update(workbook)
for document_id, attachment in sorted(attachments.items()):
digest.update(document_id.encode("ascii"))
digest.update(attachment.sha256.encode("ascii"))
return digest.hexdigest()
def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
by_okpo: dict[str, Organization] = {}
for organization in Organization.objects.filter(
okpo__in=parsed.structures, directory_imported_at__isnull=False
).only("uid", "name", "inn", "ogrn", "kpp", "okpo"):
if organization.okpo in by_okpo:
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
by_okpo[organization.okpo] = organization
return by_okpo
def _known_workbook(
parsed: ParsedWorkbook, organizations: dict[str, Organization]
) -> ParsedWorkbook:
structures = {
okpo: item for okpo, item in parsed.structures.items() if okpo in organizations
}
events = [event for item in structures.values() for event in item.events]
document_ids = {document_id for event in events for document_id in event.documents}
return ParsedWorkbook(
structures=structures,
documents={
key: value for key, value in parsed.documents.items() if key in document_ids
},
links_count=sum(len(event.documents) for event in events),
)
def _resolution_metadata(
parsed: ParsedWorkbook, organizations: dict[str, Organization]
) -> dict:
missing = sorted(set(parsed.structures) - organizations.keys())
message = "Организация не загружена: ОКПО отсутствует в каноническом реестре."
identity = "\n".join(
f"{okpo}:{organization.uid}"
for okpo, organization in sorted(organizations.items())
)
return {
"resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(),
"imported_organizations_count": len(organizations),
"skipped_organizations_count": len(missing),
"skipped_organizations": [
{"okpo": okpo, "code": "unknown_canonical_okpo", "message": message}
for okpo in missing
],
"warnings": [
{
"code": "unknown_canonical_okpo",
"message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.",
"meta": {"okpos": missing},
}
]
if missing
else [],
}
def _save_attachments(
attachments: dict[str, ValidatedAttachment],
) -> dict[str, IntegratedStructureAttachment]:
result = {}
for document_id, attachment in attachments.items():
stored = IntegratedStructureAttachment.objects.filter(
sha256=attachment.sha256
).first()
if stored is None:
stored = IntegratedStructureAttachment(
sha256=attachment.sha256, size_bytes=len(attachment.data)
)
stored.file.save(attachment.name, ContentFile(attachment.data), save=False)
stored.save()
result[document_id] = stored
return result
def _publish_relations(
artifact: ParserSourceArtifact,
parsed: ParsedWorkbook,
attachments: dict[str, ValidatedAttachment],
) -> None:
stored_attachments = _save_attachments(attachments)
documents = {}
for document_id, document in parsed.documents.items():
documents[document_id] = IntegratedStructureDocument.objects.create(
artifact=artifact,
external_id=document_id,
name=document.name,
document_type=document.document_type,
number=document.number,
document_date=document.date,
dsp=document.dsp,
have_text=document.have_text,
consultant_web_url=document.consultant_web_url or "",
attachment_filename=attachments[document_id].name
if document_id in attachments
else "",
attachment=stored_attachments.get(document_id),
)
for okpo, structure in parsed.structures.items():
external_id = f"integrated-structure:{okpo}"
record = OrganizationSourceRecord.objects.get(
uid=stable_registry_uid(RECORD_SOURCE, external_id)
)
record.integrated_events.all().delete()
for item in structure.events:
event = IntegratedStructureEvent.objects.create(
record=record,
external_id=item.external_id,
event_date=item.date,
note=item.note or "",
)
IntegratedStructureEventDocument.objects.bulk_create(
[
IntegratedStructureEventDocument(
event=event, document=documents[document_id]
)
for document_id in item.documents
]
)
def import_integrated_structures(
*,
workbook: BinaryIO,
original_name: str,
attachments: Iterable[BinaryIO],
uploaded_by_id: int | None = None,
) -> tuple[ParserSourceArtifact | None, dict]:
"""Validate all inputs and atomically publish known organizations."""
workbook.seek(0)
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
validated_attachments = _validated_attachments(attachments, parsed)
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
organizations = _resolve_organizations(parsed)
resolution = _resolution_metadata(parsed, organizations)
if not organizations:
return None, {
"status": "skipped",
"reason": "no_known_organizations",
"records_count": 0,
**resolution,
}
existing = (
ParserSourceArtifact.objects.filter(
source=PARSER_SOURCE,
status=ParserSourceArtifact.Status.PUBLISHED,
)
.order_by("-created_at")
.first()
)
if (
existing is not None
and existing.metadata.get("composite_sha256") == composite_hash
and existing.metadata.get("resolution_sha256")
== resolution["resolution_sha256"]
):
return existing, {
"status": "skipped",
"reason": "identical_batch",
"batch_id": existing.load_batch,
"records_count": existing.metadata["batch_published_records_count"],
**existing.metadata,
}
known = _known_workbook(parsed, organizations)
submitted_attachments_count = len(validated_attachments)
validated_attachments = {
key: value
for key, value in validated_attachments.items()
if key in known.documents
}
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
)
artifact = ParserSourceArtifact.objects.create(
source=PARSER_SOURCE,
original_name=original_name[:512],
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
load_batch=batch_id,
uploaded_by_id=uploaded_by_id,
version="is_stages+is_stages_docs/v1",
parsed_count=len(parsed.structures),
metadata={
"composite_sha256": composite_hash,
**resolution,
"structures_count": parsed.counts[0],
"event_rows_count": parsed.counts[1],
"document_link_rows_count": parsed.counts[2],
"unique_documents_count": parsed.counts[3],
"events_without_documents_count": parsed.counts[4],
"submitted_attachments_count": submitted_attachments_count,
"imported_documents_count": len(known.documents),
"attachments_count": len(validated_attachments),
"missing_attachments_count": len(known.documents)
- len(validated_attachments),
},
)
try:
save_artifact_file(artifact, BytesIO(workbook_bytes))
pending = []
for index, (okpo, structure) in enumerate(known.structures.items(), 1):
organization = organizations[okpo]
external_id = f"integrated-structure:{okpo}"
document_ids = {
doc_id for event in structure.events for doc_id in event.documents
}
event_dates = [event.date for event in structure.events if event.date]
pending.append(
ParserStagedRecord(
artifact=artifact,
row_number=index,
external_id=external_id,
record_type=RECORD_TYPE,
organization=organization,
normalized_data={
"title": organization.name,
"record_date": max(event_dates) if event_dates else "",
"status": "active",
"payload": {
"structure": {
"events_count": len(structure.events),
"document_links_count": sum(
len(event.documents) for event in structure.events
),
"unique_documents_count": len(document_ids),
"first_event_date": min(event_dates)
if event_dates
else "",
"latest_event_date": max(event_dates)
if event_dates
else "",
},
},
},
)
)
stage_records(pending)
artifact.status = ParserSourceArtifact.Status.PARSED
artifact.save(update_fields=["status", "updated_at"])
def finalize(artifact, result):
_publish_relations(artifact, known, validated_attachments)
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.SUCCESS,
records_count=len(organizations),
)
publish_snapshot(
artifact,
record_source=RECORD_SOURCE,
on_publish=finalize,
scope_organization_ids=[
organization.uid for organization in organizations.values()
]
if resolution["skipped_organizations_count"]
else None,
)
return artifact, {
"status": "partial"
if resolution["skipped_organizations_count"]
else "published",
"batch_id": batch_id,
"records_count": len(organizations),
**artifact.metadata,
}
except Exception as exc:
reject_snapshot(artifact, exc)
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.FAILED,
error_message=type(exc).__name__,
)
raise