feat: add Fedresurs profiles and integrated structures sources
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 6m35s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 6m35s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
306
src/apps/parsers/integrated_structures.py
Normal file
306
src/apps/parsers/integrated_structures.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""Atomic XLSX/RTF import for integrated structures."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from io import BytesIO
|
||||
from typing import BinaryIO
|
||||
|
||||
from apps.parsers.integrated_structure_workbook import (
|
||||
IntegratedWorkbookError,
|
||||
ParsedWorkbook,
|
||||
parse_integrated_workbook,
|
||||
)
|
||||
from apps.parsers.models import (
|
||||
IntegratedStructureAttachment,
|
||||
IntegratedStructureDocument,
|
||||
IntegratedStructureEvent,
|
||||
IntegratedStructureEventDocument,
|
||||
ParserLoadLog,
|
||||
ParserSourceArtifact,
|
||||
ParserStagedRecord,
|
||||
)
|
||||
from apps.parsers.registry_snapshots import (
|
||||
publish_snapshot,
|
||||
reject_snapshot,
|
||||
save_artifact_file,
|
||||
stable_registry_uid,
|
||||
stage_records,
|
||||
)
|
||||
from apps.parsers.services import ParserLoadLogService
|
||||
from django.core.files.base import ContentFile
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
|
||||
PARSER_SOURCE = "integrated_structure_documents"
|
||||
RECORD_SOURCE = "integrated_structures"
|
||||
RECORD_TYPE = "integrated_structure"
|
||||
RTF_NAME = re.compile(r"^([0-9]{1,12})\.rtf$", re.IGNORECASE)
|
||||
MAX_ATTACHMENTS = 500
|
||||
MAX_ATTACHMENT_BYTES = 5 * 1024 * 1024
|
||||
MAX_TOTAL_ATTACHMENT_BYTES = 100 * 1024 * 1024
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ValidatedAttachment:
|
||||
document_id: str
|
||||
name: str
|
||||
sha256: str
|
||||
data: bytes
|
||||
|
||||
|
||||
def _resolve_attachment_document_id(raw_id: str, parsed: ParsedWorkbook) -> str:
|
||||
if raw_id in parsed.documents:
|
||||
return raw_id
|
||||
candidates = [
|
||||
candidate for candidate in parsed.documents if int(candidate) == int(raw_id)
|
||||
]
|
||||
if len(candidates) != 1:
|
||||
raise IntegratedWorkbookError("unknown_or_ambiguous_attachment_document")
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _validated_attachments(
|
||||
files: Iterable[BinaryIO], parsed: ParsedWorkbook
|
||||
) -> dict[str, ValidatedAttachment]:
|
||||
result: dict[str, ValidatedAttachment] = {}
|
||||
total_size = 0
|
||||
for count, file in enumerate(files, 1):
|
||||
if count > MAX_ATTACHMENTS:
|
||||
raise IntegratedWorkbookError("too_many_attachments")
|
||||
name = str(getattr(file, "name", ""))
|
||||
match = RTF_NAME.fullmatch(name)
|
||||
if match is None:
|
||||
raise IntegratedWorkbookError("invalid_attachment_name")
|
||||
document_id = _resolve_attachment_document_id(match[1], parsed)
|
||||
if document_id in result:
|
||||
raise IntegratedWorkbookError("duplicate_attachment")
|
||||
if document_id not in parsed.documents:
|
||||
raise IntegratedWorkbookError("unknown_attachment_document")
|
||||
file.seek(0)
|
||||
data = file.read(MAX_ATTACHMENT_BYTES + 1)
|
||||
if len(data) > MAX_ATTACHMENT_BYTES:
|
||||
raise IntegratedWorkbookError("attachment_too_large")
|
||||
total_size += len(data)
|
||||
if total_size > MAX_TOTAL_ATTACHMENT_BYTES:
|
||||
raise IntegratedWorkbookError("attachments_too_large")
|
||||
if not data.startswith(b"{\\rtf"):
|
||||
raise IntegratedWorkbookError("invalid_rtf_signature")
|
||||
result[document_id] = ValidatedAttachment(
|
||||
document_id=document_id,
|
||||
name=name,
|
||||
sha256=hashlib.sha256(data).hexdigest(),
|
||||
data=data,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _composite_hash(
|
||||
workbook: bytes, attachments: dict[str, ValidatedAttachment]
|
||||
) -> str:
|
||||
digest = hashlib.sha256()
|
||||
digest.update(len(workbook).to_bytes(8, "big"))
|
||||
digest.update(workbook)
|
||||
for document_id, attachment in sorted(attachments.items()):
|
||||
digest.update(document_id.encode("ascii"))
|
||||
digest.update(attachment.sha256.encode("ascii"))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
|
||||
by_okpo: dict[str, Organization] = {}
|
||||
for organization in Organization.objects.filter(
|
||||
okpo__in=parsed.structures, directory_imported_at__isnull=False
|
||||
).only("uid", "name", "inn", "ogrn", "kpp", "okpo"):
|
||||
if organization.okpo in by_okpo:
|
||||
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
|
||||
by_okpo[organization.okpo] = organization
|
||||
if len(by_okpo) != len(parsed.structures):
|
||||
raise IntegratedWorkbookError("unknown_canonical_okpo")
|
||||
return by_okpo
|
||||
|
||||
|
||||
def _save_attachments(
|
||||
attachments: dict[str, ValidatedAttachment],
|
||||
) -> dict[str, IntegratedStructureAttachment]:
|
||||
result = {}
|
||||
for document_id, attachment in attachments.items():
|
||||
stored = IntegratedStructureAttachment.objects.filter(
|
||||
sha256=attachment.sha256
|
||||
).first()
|
||||
if stored is None:
|
||||
stored = IntegratedStructureAttachment(
|
||||
sha256=attachment.sha256, size_bytes=len(attachment.data)
|
||||
)
|
||||
stored.file.save(attachment.name, ContentFile(attachment.data), save=False)
|
||||
stored.save()
|
||||
result[document_id] = stored
|
||||
return result
|
||||
|
||||
|
||||
def _publish_relations(
|
||||
artifact: ParserSourceArtifact,
|
||||
parsed: ParsedWorkbook,
|
||||
attachments: dict[str, ValidatedAttachment],
|
||||
) -> None:
|
||||
stored_attachments = _save_attachments(attachments)
|
||||
documents = {}
|
||||
for document_id, document in parsed.documents.items():
|
||||
documents[document_id] = IntegratedStructureDocument.objects.create(
|
||||
artifact=artifact,
|
||||
external_id=document_id,
|
||||
name=document.name,
|
||||
document_type=document.document_type,
|
||||
number=document.number,
|
||||
document_date=document.date,
|
||||
dsp=document.dsp,
|
||||
have_text=document.have_text,
|
||||
consultant_web_url=document.consultant_web_url or "",
|
||||
attachment_filename=attachments[document_id].name
|
||||
if document_id in attachments
|
||||
else "",
|
||||
attachment=stored_attachments.get(document_id),
|
||||
)
|
||||
for okpo, structure in parsed.structures.items():
|
||||
external_id = f"integrated-structure:{okpo}"
|
||||
record = OrganizationSourceRecord.objects.get(
|
||||
uid=stable_registry_uid(RECORD_SOURCE, external_id)
|
||||
)
|
||||
record.integrated_events.all().delete()
|
||||
for item in structure.events:
|
||||
event = IntegratedStructureEvent.objects.create(
|
||||
record=record,
|
||||
external_id=item.external_id,
|
||||
event_date=item.date,
|
||||
note=item.note or "",
|
||||
)
|
||||
IntegratedStructureEventDocument.objects.bulk_create(
|
||||
[
|
||||
IntegratedStructureEventDocument(
|
||||
event=event, document=documents[document_id]
|
||||
)
|
||||
for document_id in item.documents
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def import_integrated_structures(
|
||||
*,
|
||||
workbook: BinaryIO,
|
||||
original_name: str,
|
||||
attachments: Iterable[BinaryIO],
|
||||
uploaded_by_id: int | None = None,
|
||||
) -> tuple[ParserSourceArtifact, dict]:
|
||||
"""Validate all inputs, stage all structures, then publish one transaction."""
|
||||
workbook.seek(0)
|
||||
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
|
||||
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
|
||||
validated_attachments = _validated_attachments(attachments, parsed)
|
||||
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
|
||||
existing = (
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source=PARSER_SOURCE,
|
||||
status=ParserSourceArtifact.Status.PUBLISHED,
|
||||
metadata__composite_sha256=composite_hash,
|
||||
)
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
return existing, {
|
||||
"status": "skipped",
|
||||
"reason": "identical_batch",
|
||||
"batch_id": existing.load_batch,
|
||||
"records_count": existing.published_count,
|
||||
**existing.metadata,
|
||||
}
|
||||
organizations = _resolve_organizations(parsed)
|
||||
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
||||
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
|
||||
)
|
||||
artifact = ParserSourceArtifact.objects.create(
|
||||
source=PARSER_SOURCE,
|
||||
original_name=original_name[:512],
|
||||
content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
load_batch=batch_id,
|
||||
uploaded_by_id=uploaded_by_id,
|
||||
version="is_stages+is_stages_docs/v1",
|
||||
parsed_count=len(parsed.structures),
|
||||
metadata={
|
||||
"composite_sha256": composite_hash,
|
||||
"structures_count": parsed.counts[0],
|
||||
"event_rows_count": parsed.counts[1],
|
||||
"document_link_rows_count": parsed.counts[2],
|
||||
"unique_documents_count": parsed.counts[3],
|
||||
"events_without_documents_count": parsed.counts[4],
|
||||
"attachments_count": len(validated_attachments),
|
||||
"missing_attachments_count": len(parsed.documents)
|
||||
- len(validated_attachments),
|
||||
},
|
||||
)
|
||||
try:
|
||||
save_artifact_file(artifact, BytesIO(workbook_bytes))
|
||||
pending = []
|
||||
for index, (okpo, structure) in enumerate(parsed.structures.items(), 1):
|
||||
organization = organizations[okpo]
|
||||
external_id = f"integrated-structure:{okpo}"
|
||||
document_ids = {
|
||||
doc_id for event in structure.events for doc_id in event.documents
|
||||
}
|
||||
event_dates = [event.date for event in structure.events if event.date]
|
||||
pending.append(
|
||||
ParserStagedRecord(
|
||||
artifact=artifact,
|
||||
row_number=index,
|
||||
external_id=external_id,
|
||||
record_type=RECORD_TYPE,
|
||||
organization=organization,
|
||||
normalized_data={
|
||||
"title": organization.name,
|
||||
"record_date": max(event_dates) if event_dates else "",
|
||||
"status": "active",
|
||||
"payload": {
|
||||
"structure": {
|
||||
"events_count": len(structure.events),
|
||||
"document_links_count": sum(
|
||||
len(event.documents) for event in structure.events
|
||||
),
|
||||
"unique_documents_count": len(document_ids),
|
||||
"first_event_date": min(event_dates) if event_dates else "",
|
||||
"latest_event_date": max(event_dates) if event_dates else "",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
stage_records(pending)
|
||||
artifact.status = ParserSourceArtifact.Status.PARSED
|
||||
artifact.save(update_fields=["status", "updated_at"])
|
||||
|
||||
def finalize(artifact, result):
|
||||
_publish_relations(artifact, parsed, validated_attachments)
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.SUCCESS,
|
||||
records_count=result.published,
|
||||
)
|
||||
|
||||
result = publish_snapshot(
|
||||
artifact, record_source=RECORD_SOURCE, on_publish=finalize
|
||||
)
|
||||
return artifact, {
|
||||
"status": "published",
|
||||
"batch_id": batch_id,
|
||||
"records_count": result.published,
|
||||
**artifact.metadata,
|
||||
}
|
||||
except Exception as exc:
|
||||
reject_snapshot(artifact, exc)
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.FAILED,
|
||||
error_message=type(exc).__name__,
|
||||
)
|
||||
raise
|
||||
Reference in New Issue
Block a user