fix(import): skip unknown organizations in integrated workbooks

This commit is contained in:
Aleksandr Meshchryakov
2026-09-28 11:27:46 +02:00
parent ecf3855aa7
commit 262193f121
4 changed files with 441 additions and 18 deletions

View File

@@ -117,11 +117,55 @@ def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
if organization.okpo in by_okpo:
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
by_okpo[organization.okpo] = organization
if len(by_okpo) != len(parsed.structures):
raise IntegratedWorkbookError("unknown_canonical_okpo")
return by_okpo
def _known_workbook(
parsed: ParsedWorkbook, organizations: dict[str, Organization]
) -> ParsedWorkbook:
structures = {
okpo: item for okpo, item in parsed.structures.items() if okpo in organizations
}
events = [event for item in structures.values() for event in item.events]
document_ids = {document_id for event in events for document_id in event.documents}
return ParsedWorkbook(
structures=structures,
documents={
key: value for key, value in parsed.documents.items() if key in document_ids
},
links_count=sum(len(event.documents) for event in events),
)
def _resolution_metadata(
parsed: ParsedWorkbook, organizations: dict[str, Organization]
) -> dict:
missing = sorted(set(parsed.structures) - organizations.keys())
message = "Организация не загружена: ОКПО отсутствует в каноническом реестре."
identity = "\n".join(
f"{okpo}:{organization.uid}"
for okpo, organization in sorted(organizations.items())
)
return {
"resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(),
"imported_organizations_count": len(organizations),
"skipped_organizations_count": len(missing),
"skipped_organizations": [
{"okpo": okpo, "code": "unknown_canonical_okpo", "message": message}
for okpo in missing
],
"warnings": [
{
"code": "unknown_canonical_okpo",
"message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.",
"meta": {"okpos": missing},
}
]
if missing
else [],
}
def _save_attachments(
attachments: dict[str, ValidatedAttachment],
) -> dict[str, IntegratedStructureAttachment]:
@@ -192,31 +236,50 @@ def import_integrated_structures(
original_name: str,
attachments: Iterable[BinaryIO],
uploaded_by_id: int | None = None,
) -> tuple[ParserSourceArtifact, dict]:
"""Validate all inputs, stage all structures, then publish one transaction."""
) -> tuple[ParserSourceArtifact | None, dict]:
"""Validate all inputs and atomically publish known organizations."""
workbook.seek(0)
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
validated_attachments = _validated_attachments(attachments, parsed)
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
organizations = _resolve_organizations(parsed)
resolution = _resolution_metadata(parsed, organizations)
if not organizations:
return None, {
"status": "skipped",
"reason": "no_known_organizations",
"records_count": 0,
**resolution,
}
existing = (
ParserSourceArtifact.objects.filter(
source=PARSER_SOURCE,
status=ParserSourceArtifact.Status.PUBLISHED,
metadata__composite_sha256=composite_hash,
)
.order_by("-created_at")
.first()
)
if existing is not None:
if (
existing is not None
and existing.metadata.get("composite_sha256") == composite_hash
and existing.metadata.get("resolution_sha256")
== resolution["resolution_sha256"]
):
return existing, {
"status": "skipped",
"reason": "identical_batch",
"batch_id": existing.load_batch,
"records_count": existing.published_count,
"records_count": existing.metadata["batch_published_records_count"],
**existing.metadata,
}
organizations = _resolve_organizations(parsed)
known = _known_workbook(parsed, organizations)
submitted_attachments_count = len(validated_attachments)
validated_attachments = {
key: value
for key, value in validated_attachments.items()
if key in known.documents
}
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
)
@@ -230,20 +293,23 @@ def import_integrated_structures(
parsed_count=len(parsed.structures),
metadata={
"composite_sha256": composite_hash,
**resolution,
"structures_count": parsed.counts[0],
"event_rows_count": parsed.counts[1],
"document_link_rows_count": parsed.counts[2],
"unique_documents_count": parsed.counts[3],
"events_without_documents_count": parsed.counts[4],
"submitted_attachments_count": submitted_attachments_count,
"imported_documents_count": len(known.documents),
"attachments_count": len(validated_attachments),
"missing_attachments_count": len(parsed.documents)
"missing_attachments_count": len(known.documents)
- len(validated_attachments),
},
)
try:
save_artifact_file(artifact, BytesIO(workbook_bytes))
pending = []
for index, (okpo, structure) in enumerate(parsed.structures.items(), 1):
for index, (okpo, structure) in enumerate(known.structures.items(), 1):
organization = organizations[okpo]
external_id = f"integrated-structure:{okpo}"
document_ids = {
@@ -284,20 +350,29 @@ def import_integrated_structures(
artifact.save(update_fields=["status", "updated_at"])
def finalize(artifact, result):
_publish_relations(artifact, parsed, validated_attachments)
_publish_relations(artifact, known, validated_attachments)
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.SUCCESS,
records_count=result.published,
records_count=len(organizations),
)
result = publish_snapshot(
artifact, record_source=RECORD_SOURCE, on_publish=finalize
publish_snapshot(
artifact,
record_source=RECORD_SOURCE,
on_publish=finalize,
scope_organization_ids=[
organization.uid for organization in organizations.values()
]
if resolution["skipped_organizations_count"]
else None,
)
return artifact, {
"status": "published",
"status": "partial"
if resolution["skipped_organizations_count"]
else "published",
"batch_id": batch_id,
"records_count": result.published,
"records_count": len(organizations),
**artifact.metadata,
}
except Exception as exc:

View File

@@ -3171,9 +3171,10 @@ class ParserUploadView(APIView):
except Exception as exc:
job.fail(error=type(exc).__name__)
raise
job.complete(result={**result, "artifact_id": str(artifact.uid)})
artifact_id = str(artifact.uid) if artifact is not None else None
job.complete(result={**result, "artifact_id": artifact_id})
return api_response(
{**result, "artifact_id": str(artifact.uid), "task_id": job.task_id},
{**result, "artifact_id": artifact_id, "task_id": job.task_id},
status_code=status.HTTP_200_OK,
)
if not descriptor.supports_file_upload: