"""Atomic XLSX/RTF import for integrated structures.""" from __future__ import annotations import hashlib import re from collections.abc import Iterable from dataclasses import dataclass from io import BytesIO from typing import BinaryIO from apps.parsers.integrated_structure_workbook import ( IntegratedWorkbookError, ParsedWorkbook, parse_integrated_workbook, ) from apps.parsers.models import ( IntegratedStructureAttachment, IntegratedStructureDocument, IntegratedStructureEvent, IntegratedStructureEventDocument, ParserLoadLog, ParserSourceArtifact, ParserStagedRecord, ) from apps.parsers.registry_snapshots import ( publish_snapshot, reject_snapshot, save_artifact_file, stable_registry_uid, stage_records, ) from apps.parsers.services import ParserLoadLogService from django.core.files.base import ContentFile from organizations.models import Organization, OrganizationSourceRecord PARSER_SOURCE = "integrated_structure_documents" RECORD_SOURCE = "integrated_structures" RECORD_TYPE = "integrated_structure" RTF_NAME = re.compile(r"^([0-9]{1,12})\.rtf$", re.IGNORECASE) MAX_ATTACHMENTS = 500 MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024 MAX_TOTAL_ATTACHMENT_BYTES = 100 * 1024 * 1024 @dataclass(frozen=True) class ValidatedAttachment: document_id: str name: str sha256: str data: bytes def _resolve_attachment_document_id(raw_id: str, parsed: ParsedWorkbook) -> str: if raw_id in parsed.documents: return raw_id candidates = [ candidate for candidate in parsed.documents if int(candidate) == int(raw_id) ] if len(candidates) != 1: raise IntegratedWorkbookError("unknown_or_ambiguous_attachment_document") return candidates[0] def _validated_attachments( files: Iterable[BinaryIO], parsed: ParsedWorkbook ) -> dict[str, ValidatedAttachment]: result: dict[str, ValidatedAttachment] = {} total_size = 0 for count, file in enumerate(files, 1): if count > MAX_ATTACHMENTS: raise IntegratedWorkbookError("too_many_attachments") name = str(getattr(file, "name", "")) match = RTF_NAME.fullmatch(name) if match is None: raise IntegratedWorkbookError("invalid_attachment_name") document_id = _resolve_attachment_document_id(match[1], parsed) if document_id in result: raise IntegratedWorkbookError("duplicate_attachment") if document_id not in parsed.documents: raise IntegratedWorkbookError("unknown_attachment_document") file.seek(0) data = file.read(MAX_ATTACHMENT_BYTES + 1) if len(data) > MAX_ATTACHMENT_BYTES: raise IntegratedWorkbookError("attachment_too_large") total_size += len(data) if total_size > MAX_TOTAL_ATTACHMENT_BYTES: raise IntegratedWorkbookError("attachments_too_large") if not data.startswith(b"{\\rtf"): raise IntegratedWorkbookError("invalid_rtf_signature") result[document_id] = ValidatedAttachment( document_id=document_id, name=name, sha256=hashlib.sha256(data).hexdigest(), data=data, ) return result def _composite_hash( workbook: bytes, attachments: dict[str, ValidatedAttachment] ) -> str: digest = hashlib.sha256() digest.update(len(workbook).to_bytes(8, "big")) digest.update(workbook) for document_id, attachment in sorted(attachments.items()): digest.update(document_id.encode("ascii")) digest.update(attachment.sha256.encode("ascii")) return digest.hexdigest() def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]: by_okpo: dict[str, Organization] = {} for organization in Organization.objects.filter( okpo__in=parsed.structures, directory_imported_at__isnull=False ).only("uid", "name", "inn", "ogrn", "kpp", "okpo"): if organization.okpo in by_okpo: raise IntegratedWorkbookError("ambiguous_canonical_okpo") by_okpo[organization.okpo] = organization return by_okpo def _known_workbook( parsed: ParsedWorkbook, organizations: dict[str, Organization] ) -> ParsedWorkbook: structures = { okpo: item for okpo, item in parsed.structures.items() if okpo in organizations } events = [event for item in structures.values() for event in item.events] document_ids = {document_id for event in events for document_id in event.documents} return ParsedWorkbook( structures=structures, documents={ key: value for key, value in parsed.documents.items() if key in document_ids }, links_count=sum(len(event.documents) for event in events), ) def _resolution_metadata( parsed: ParsedWorkbook, organizations: dict[str, Organization] ) -> dict: missing = sorted(set(parsed.structures) - organizations.keys()) message = "Организация не загружена: ОКПО отсутствует в каноническом реестре." identity = "\n".join( f"{okpo}:{organization.uid}" for okpo, organization in sorted(organizations.items()) ) return { "resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(), "imported_organizations_count": len(organizations), "skipped_organizations_count": len(missing), "skipped_organizations": [ {"okpo": okpo, "code": "unknown_canonical_okpo", "message": message} for okpo in missing ], "warnings": [ { "code": "unknown_canonical_okpo", "message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.", "meta": {"okpos": missing}, } ] if missing else [], } def _save_attachments( attachments: dict[str, ValidatedAttachment], ) -> dict[str, IntegratedStructureAttachment]: result = {} for document_id, attachment in attachments.items(): stored = IntegratedStructureAttachment.objects.filter( sha256=attachment.sha256 ).first() if stored is None: stored = IntegratedStructureAttachment( sha256=attachment.sha256, size_bytes=len(attachment.data) ) stored.file.save(attachment.name, ContentFile(attachment.data), save=False) stored.save() result[document_id] = stored return result def _publish_relations( artifact: ParserSourceArtifact, parsed: ParsedWorkbook, attachments: dict[str, ValidatedAttachment], ) -> None: stored_attachments = _save_attachments(attachments) documents = {} for document_id, document in parsed.documents.items(): documents[document_id] = IntegratedStructureDocument.objects.create( artifact=artifact, external_id=document_id, name=document.name, document_type=document.document_type, number=document.number, document_date=document.date, dsp=document.dsp, have_text=document.have_text, consultant_web_url=document.consultant_web_url or "", attachment_filename=attachments[document_id].name if document_id in attachments else "", attachment=stored_attachments.get(document_id), ) for okpo, structure in parsed.structures.items(): external_id = f"integrated-structure:{okpo}" record = OrganizationSourceRecord.objects.get( uid=stable_registry_uid(RECORD_SOURCE, external_id) ) record.integrated_events.all().delete() for item in structure.events: event = IntegratedStructureEvent.objects.create( record=record, external_id=item.external_id, event_date=item.date, note=item.note or "", ) IntegratedStructureEventDocument.objects.bulk_create( [ IntegratedStructureEventDocument( event=event, document=documents[document_id] ) for document_id in item.documents ] ) def import_integrated_structures( *, workbook: BinaryIO, original_name: str, attachments: Iterable[BinaryIO], uploaded_by_id: int | None = None, ) -> tuple[ParserSourceArtifact | None, dict]: """Validate all inputs and atomically publish known organizations.""" workbook.seek(0) workbook_bytes = workbook.read(20 * 1024 * 1024 + 1) parsed = parse_integrated_workbook(BytesIO(workbook_bytes)) validated_attachments = _validated_attachments(attachments, parsed) composite_hash = _composite_hash(workbook_bytes, validated_attachments) organizations = _resolve_organizations(parsed) resolution = _resolution_metadata(parsed, organizations) if not organizations: return None, { "status": "skipped", "reason": "no_known_organizations", "records_count": 0, **resolution, } existing = ( ParserSourceArtifact.objects.filter( source=PARSER_SOURCE, status=ParserSourceArtifact.Status.PUBLISHED, ) .order_by("-created_at") .first() ) if ( existing is not None and existing.metadata.get("composite_sha256") == composite_hash and existing.metadata.get("resolution_sha256") == resolution["resolution_sha256"] ): return existing, { "status": "skipped", "reason": "identical_batch", "batch_id": existing.load_batch, "records_count": existing.metadata["batch_published_records_count"], **existing.metadata, } known = _known_workbook(parsed, organizations) submitted_attachments_count = len(validated_attachments) validated_attachments = { key: value for key, value in validated_attachments.items() if key in known.documents } load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id( source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS ) artifact = ParserSourceArtifact.objects.create( source=PARSER_SOURCE, original_name=original_name[:512], content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", load_batch=batch_id, uploaded_by_id=uploaded_by_id, version="is_stages+is_stages_docs/v1", parsed_count=len(parsed.structures), metadata={ "composite_sha256": composite_hash, **resolution, "structures_count": parsed.counts[0], "event_rows_count": parsed.counts[1], "document_link_rows_count": parsed.counts[2], "unique_documents_count": parsed.counts[3], "events_without_documents_count": parsed.counts[4], "submitted_attachments_count": submitted_attachments_count, "imported_documents_count": len(known.documents), "attachments_count": len(validated_attachments), "missing_attachments_count": len(known.documents) - len(validated_attachments), }, ) try: save_artifact_file(artifact, BytesIO(workbook_bytes)) pending = [] for index, (okpo, structure) in enumerate(known.structures.items(), 1): organization = organizations[okpo] external_id = f"integrated-structure:{okpo}" document_ids = { doc_id for event in structure.events for doc_id in event.documents } event_dates = [event.date for event in structure.events if event.date] pending.append( ParserStagedRecord( artifact=artifact, row_number=index, external_id=external_id, record_type=RECORD_TYPE, organization=organization, normalized_data={ "title": organization.name, "record_date": max(event_dates) if event_dates else "", "status": "active", "payload": { "structure": { "events_count": len(structure.events), "document_links_count": sum( len(event.documents) for event in structure.events ), "unique_documents_count": len(document_ids), "first_event_date": min(event_dates) if event_dates else "", "latest_event_date": max(event_dates) if event_dates else "", }, }, }, ) ) stage_records(pending) artifact.status = ParserSourceArtifact.Status.PARSED artifact.save(update_fields=["status", "updated_at"]) def finalize(artifact, result): _publish_relations(artifact, known, validated_attachments) ParserLoadLogService.update( load_log, status=ParserLoadLog.Status.SUCCESS, records_count=len(organizations), ) publish_snapshot( artifact, record_source=RECORD_SOURCE, on_publish=finalize, scope_organization_ids=[ organization.uid for organization in organizations.values() ] if resolution["skipped_organizations_count"] else None, ) return artifact, { "status": "partial" if resolution["skipped_organizations_count"] else "published", "batch_id": batch_id, "records_count": len(organizations), **artifact.metadata, } except Exception as exc: reject_snapshot(artifact, exc) ParserLoadLogService.update( load_log, status=ParserLoadLog.Status.FAILED, error_message=type(exc).__name__, ) raise