386 lines
14 KiB
Python
386 lines
14 KiB
Python
"""Atomic XLSX/RTF import for integrated structures."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import re
|
|
from collections.abc import Iterable
|
|
from dataclasses import dataclass
|
|
from io import BytesIO
|
|
from typing import BinaryIO
|
|
|
|
from apps.parsers.integrated_structure_workbook import (
|
|
IntegratedWorkbookError,
|
|
ParsedWorkbook,
|
|
parse_integrated_workbook,
|
|
)
|
|
from apps.parsers.models import (
|
|
IntegratedStructureAttachment,
|
|
IntegratedStructureDocument,
|
|
IntegratedStructureEvent,
|
|
IntegratedStructureEventDocument,
|
|
ParserLoadLog,
|
|
ParserSourceArtifact,
|
|
ParserStagedRecord,
|
|
)
|
|
from apps.parsers.registry_snapshots import (
|
|
publish_snapshot,
|
|
reject_snapshot,
|
|
save_artifact_file,
|
|
stable_registry_uid,
|
|
stage_records,
|
|
)
|
|
from apps.parsers.services import ParserLoadLogService
|
|
from django.core.files.base import ContentFile
|
|
from organizations.models import Organization, OrganizationSourceRecord
|
|
|
|
PARSER_SOURCE = "integrated_structure_documents"
|
|
RECORD_SOURCE = "integrated_structures"
|
|
RECORD_TYPE = "integrated_structure"
|
|
RTF_NAME = re.compile(r"^([0-9]{1,12})\.rtf$", re.IGNORECASE)
|
|
MAX_ATTACHMENTS = 500
|
|
MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024
|
|
MAX_TOTAL_ATTACHMENT_BYTES = 100 * 1024 * 1024
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ValidatedAttachment:
|
|
document_id: str
|
|
name: str
|
|
sha256: str
|
|
data: bytes
|
|
|
|
|
|
def _resolve_attachment_document_id(raw_id: str, parsed: ParsedWorkbook) -> str:
|
|
if raw_id in parsed.documents:
|
|
return raw_id
|
|
candidates = [
|
|
candidate for candidate in parsed.documents if int(candidate) == int(raw_id)
|
|
]
|
|
if len(candidates) != 1:
|
|
raise IntegratedWorkbookError("unknown_or_ambiguous_attachment_document")
|
|
return candidates[0]
|
|
|
|
|
|
def _validated_attachments(
|
|
files: Iterable[BinaryIO], parsed: ParsedWorkbook
|
|
) -> dict[str, ValidatedAttachment]:
|
|
result: dict[str, ValidatedAttachment] = {}
|
|
total_size = 0
|
|
for count, file in enumerate(files, 1):
|
|
if count > MAX_ATTACHMENTS:
|
|
raise IntegratedWorkbookError("too_many_attachments")
|
|
name = str(getattr(file, "name", ""))
|
|
match = RTF_NAME.fullmatch(name)
|
|
if match is None:
|
|
raise IntegratedWorkbookError("invalid_attachment_name")
|
|
document_id = _resolve_attachment_document_id(match[1], parsed)
|
|
if document_id in result:
|
|
raise IntegratedWorkbookError("duplicate_attachment")
|
|
if document_id not in parsed.documents:
|
|
raise IntegratedWorkbookError("unknown_attachment_document")
|
|
file.seek(0)
|
|
data = file.read(MAX_ATTACHMENT_BYTES + 1)
|
|
if len(data) > MAX_ATTACHMENT_BYTES:
|
|
raise IntegratedWorkbookError("attachment_too_large")
|
|
total_size += len(data)
|
|
if total_size > MAX_TOTAL_ATTACHMENT_BYTES:
|
|
raise IntegratedWorkbookError("attachments_too_large")
|
|
if not data.startswith(b"{\\rtf"):
|
|
raise IntegratedWorkbookError("invalid_rtf_signature")
|
|
result[document_id] = ValidatedAttachment(
|
|
document_id=document_id,
|
|
name=name,
|
|
sha256=hashlib.sha256(data).hexdigest(),
|
|
data=data,
|
|
)
|
|
return result
|
|
|
|
|
|
def _composite_hash(
|
|
workbook: bytes, attachments: dict[str, ValidatedAttachment]
|
|
) -> str:
|
|
digest = hashlib.sha256()
|
|
digest.update(len(workbook).to_bytes(8, "big"))
|
|
digest.update(workbook)
|
|
for document_id, attachment in sorted(attachments.items()):
|
|
digest.update(document_id.encode("ascii"))
|
|
digest.update(attachment.sha256.encode("ascii"))
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
|
|
by_okpo: dict[str, Organization] = {}
|
|
for organization in Organization.objects.filter(
|
|
okpo__in=parsed.structures, directory_imported_at__isnull=False
|
|
).only("uid", "name", "inn", "ogrn", "kpp", "okpo"):
|
|
if organization.okpo in by_okpo:
|
|
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
|
|
by_okpo[organization.okpo] = organization
|
|
return by_okpo
|
|
|
|
|
|
def _known_workbook(
|
|
parsed: ParsedWorkbook, organizations: dict[str, Organization]
|
|
) -> ParsedWorkbook:
|
|
structures = {
|
|
okpo: item for okpo, item in parsed.structures.items() if okpo in organizations
|
|
}
|
|
events = [event for item in structures.values() for event in item.events]
|
|
document_ids = {document_id for event in events for document_id in event.documents}
|
|
return ParsedWorkbook(
|
|
structures=structures,
|
|
documents={
|
|
key: value for key, value in parsed.documents.items() if key in document_ids
|
|
},
|
|
links_count=sum(len(event.documents) for event in events),
|
|
)
|
|
|
|
|
|
def _resolution_metadata(
|
|
parsed: ParsedWorkbook, organizations: dict[str, Organization]
|
|
) -> dict:
|
|
missing = sorted(set(parsed.structures) - organizations.keys())
|
|
message = "Организация не загружена: ОКПО отсутствует в каноническом реестре."
|
|
identity = "\n".join(
|
|
f"{okpo}:{organization.uid}"
|
|
for okpo, organization in sorted(organizations.items())
|
|
)
|
|
return {
|
|
"resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(),
|
|
"imported_organizations_count": len(organizations),
|
|
"skipped_organizations_count": len(missing),
|
|
"skipped_organizations": [
|
|
{"okpo": okpo, "code": "unknown_canonical_okpo", "message": message}
|
|
for okpo in missing
|
|
],
|
|
"warnings": [
|
|
{
|
|
"code": "unknown_canonical_okpo",
|
|
"message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.",
|
|
"meta": {"okpos": missing},
|
|
}
|
|
]
|
|
if missing
|
|
else [],
|
|
}
|
|
|
|
|
|
def _save_attachments(
|
|
attachments: dict[str, ValidatedAttachment],
|
|
) -> dict[str, IntegratedStructureAttachment]:
|
|
result = {}
|
|
for document_id, attachment in attachments.items():
|
|
stored = IntegratedStructureAttachment.objects.filter(
|
|
sha256=attachment.sha256
|
|
).first()
|
|
if stored is None:
|
|
stored = IntegratedStructureAttachment(
|
|
sha256=attachment.sha256, size_bytes=len(attachment.data)
|
|
)
|
|
stored.file.save(attachment.name, ContentFile(attachment.data), save=False)
|
|
stored.save()
|
|
result[document_id] = stored
|
|
return result
|
|
|
|
|
|
def _publish_relations(
|
|
artifact: ParserSourceArtifact,
|
|
parsed: ParsedWorkbook,
|
|
attachments: dict[str, ValidatedAttachment],
|
|
) -> None:
|
|
stored_attachments = _save_attachments(attachments)
|
|
documents = {}
|
|
for document_id, document in parsed.documents.items():
|
|
documents[document_id] = IntegratedStructureDocument.objects.create(
|
|
artifact=artifact,
|
|
external_id=document_id,
|
|
name=document.name,
|
|
document_type=document.document_type,
|
|
number=document.number,
|
|
document_date=document.date,
|
|
dsp=document.dsp,
|
|
have_text=document.have_text,
|
|
consultant_web_url=document.consultant_web_url or "",
|
|
attachment_filename=attachments[document_id].name
|
|
if document_id in attachments
|
|
else "",
|
|
attachment=stored_attachments.get(document_id),
|
|
)
|
|
for okpo, structure in parsed.structures.items():
|
|
external_id = f"integrated-structure:{okpo}"
|
|
record = OrganizationSourceRecord.objects.get(
|
|
uid=stable_registry_uid(RECORD_SOURCE, external_id)
|
|
)
|
|
record.integrated_events.all().delete()
|
|
for item in structure.events:
|
|
event = IntegratedStructureEvent.objects.create(
|
|
record=record,
|
|
external_id=item.external_id,
|
|
event_date=item.date,
|
|
note=item.note or "",
|
|
)
|
|
IntegratedStructureEventDocument.objects.bulk_create(
|
|
[
|
|
IntegratedStructureEventDocument(
|
|
event=event, document=documents[document_id]
|
|
)
|
|
for document_id in item.documents
|
|
]
|
|
)
|
|
|
|
|
|
def import_integrated_structures(
|
|
*,
|
|
workbook: BinaryIO,
|
|
original_name: str,
|
|
attachments: Iterable[BinaryIO],
|
|
uploaded_by_id: int | None = None,
|
|
) -> tuple[ParserSourceArtifact | None, dict]:
|
|
"""Validate all inputs and atomically publish known organizations."""
|
|
workbook.seek(0)
|
|
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
|
|
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
|
|
validated_attachments = _validated_attachments(attachments, parsed)
|
|
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
|
|
organizations = _resolve_organizations(parsed)
|
|
resolution = _resolution_metadata(parsed, organizations)
|
|
if not organizations:
|
|
return None, {
|
|
"status": "skipped",
|
|
"reason": "no_known_organizations",
|
|
"records_count": 0,
|
|
**resolution,
|
|
}
|
|
existing = (
|
|
ParserSourceArtifact.objects.filter(
|
|
source=PARSER_SOURCE,
|
|
status=ParserSourceArtifact.Status.PUBLISHED,
|
|
)
|
|
.order_by("-created_at")
|
|
.first()
|
|
)
|
|
if (
|
|
existing is not None
|
|
and existing.metadata.get("composite_sha256") == composite_hash
|
|
and existing.metadata.get("resolution_sha256")
|
|
== resolution["resolution_sha256"]
|
|
):
|
|
return existing, {
|
|
"status": "skipped",
|
|
"reason": "identical_batch",
|
|
"batch_id": existing.load_batch,
|
|
"records_count": existing.metadata["batch_published_records_count"],
|
|
**existing.metadata,
|
|
}
|
|
known = _known_workbook(parsed, organizations)
|
|
submitted_attachments_count = len(validated_attachments)
|
|
validated_attachments = {
|
|
key: value
|
|
for key, value in validated_attachments.items()
|
|
if key in known.documents
|
|
}
|
|
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
|
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
|
|
)
|
|
artifact = ParserSourceArtifact.objects.create(
|
|
source=PARSER_SOURCE,
|
|
original_name=original_name[:512],
|
|
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
load_batch=batch_id,
|
|
uploaded_by_id=uploaded_by_id,
|
|
version="is_stages+is_stages_docs/v1",
|
|
parsed_count=len(parsed.structures),
|
|
metadata={
|
|
"composite_sha256": composite_hash,
|
|
**resolution,
|
|
"structures_count": parsed.counts[0],
|
|
"event_rows_count": parsed.counts[1],
|
|
"document_link_rows_count": parsed.counts[2],
|
|
"unique_documents_count": parsed.counts[3],
|
|
"events_without_documents_count": parsed.counts[4],
|
|
"submitted_attachments_count": submitted_attachments_count,
|
|
"imported_documents_count": len(known.documents),
|
|
"attachments_count": len(validated_attachments),
|
|
"missing_attachments_count": len(known.documents)
|
|
- len(validated_attachments),
|
|
},
|
|
)
|
|
try:
|
|
save_artifact_file(artifact, BytesIO(workbook_bytes))
|
|
pending = []
|
|
for index, (okpo, structure) in enumerate(known.structures.items(), 1):
|
|
organization = organizations[okpo]
|
|
external_id = f"integrated-structure:{okpo}"
|
|
document_ids = {
|
|
doc_id for event in structure.events for doc_id in event.documents
|
|
}
|
|
event_dates = [event.date for event in structure.events if event.date]
|
|
pending.append(
|
|
ParserStagedRecord(
|
|
artifact=artifact,
|
|
row_number=index,
|
|
external_id=external_id,
|
|
record_type=RECORD_TYPE,
|
|
organization=organization,
|
|
normalized_data={
|
|
"title": organization.name,
|
|
"record_date": max(event_dates) if event_dates else "",
|
|
"status": "active",
|
|
"payload": {
|
|
"structure": {
|
|
"events_count": len(structure.events),
|
|
"document_links_count": sum(
|
|
len(event.documents) for event in structure.events
|
|
),
|
|
"unique_documents_count": len(document_ids),
|
|
"first_event_date": min(event_dates)
|
|
if event_dates
|
|
else "",
|
|
"latest_event_date": max(event_dates)
|
|
if event_dates
|
|
else "",
|
|
},
|
|
},
|
|
},
|
|
)
|
|
)
|
|
stage_records(pending)
|
|
artifact.status = ParserSourceArtifact.Status.PARSED
|
|
artifact.save(update_fields=["status", "updated_at"])
|
|
|
|
def finalize(artifact, result):
|
|
_publish_relations(artifact, known, validated_attachments)
|
|
ParserLoadLogService.update(
|
|
load_log,
|
|
status=ParserLoadLog.Status.SUCCESS,
|
|
records_count=len(organizations),
|
|
)
|
|
|
|
publish_snapshot(
|
|
artifact,
|
|
record_source=RECORD_SOURCE,
|
|
on_publish=finalize,
|
|
scope_organization_ids=[
|
|
organization.uid for organization in organizations.values()
|
|
]
|
|
if resolution["skipped_organizations_count"]
|
|
else None,
|
|
)
|
|
return artifact, {
|
|
"status": "partial"
|
|
if resolution["skipped_organizations_count"]
|
|
else "published",
|
|
"batch_id": batch_id,
|
|
"records_count": len(organizations),
|
|
**artifact.metadata,
|
|
}
|
|
except Exception as exc:
|
|
reject_snapshot(artifact, exc)
|
|
ParserLoadLogService.update(
|
|
load_log,
|
|
status=ParserLoadLog.Status.FAILED,
|
|
error_message=type(exc).__name__,
|
|
)
|
|
raise
|