fix(import): skip unknown organizations in integrated workbooks
This commit is contained in:
@@ -117,11 +117,55 @@ def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
|
||||
if organization.okpo in by_okpo:
|
||||
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
|
||||
by_okpo[organization.okpo] = organization
|
||||
if len(by_okpo) != len(parsed.structures):
|
||||
raise IntegratedWorkbookError("unknown_canonical_okpo")
|
||||
return by_okpo
|
||||
|
||||
|
||||
def _known_workbook(
|
||||
parsed: ParsedWorkbook, organizations: dict[str, Organization]
|
||||
) -> ParsedWorkbook:
|
||||
structures = {
|
||||
okpo: item for okpo, item in parsed.structures.items() if okpo in organizations
|
||||
}
|
||||
events = [event for item in structures.values() for event in item.events]
|
||||
document_ids = {document_id for event in events for document_id in event.documents}
|
||||
return ParsedWorkbook(
|
||||
structures=structures,
|
||||
documents={
|
||||
key: value for key, value in parsed.documents.items() if key in document_ids
|
||||
},
|
||||
links_count=sum(len(event.documents) for event in events),
|
||||
)
|
||||
|
||||
|
||||
def _resolution_metadata(
|
||||
parsed: ParsedWorkbook, organizations: dict[str, Organization]
|
||||
) -> dict:
|
||||
missing = sorted(set(parsed.structures) - organizations.keys())
|
||||
message = "Организация не загружена: ОКПО отсутствует в каноническом реестре."
|
||||
identity = "\n".join(
|
||||
f"{okpo}:{organization.uid}"
|
||||
for okpo, organization in sorted(organizations.items())
|
||||
)
|
||||
return {
|
||||
"resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(),
|
||||
"imported_organizations_count": len(organizations),
|
||||
"skipped_organizations_count": len(missing),
|
||||
"skipped_organizations": [
|
||||
{"okpo": okpo, "code": "unknown_canonical_okpo", "message": message}
|
||||
for okpo in missing
|
||||
],
|
||||
"warnings": [
|
||||
{
|
||||
"code": "unknown_canonical_okpo",
|
||||
"message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.",
|
||||
"meta": {"okpos": missing},
|
||||
}
|
||||
]
|
||||
if missing
|
||||
else [],
|
||||
}
|
||||
|
||||
|
||||
def _save_attachments(
|
||||
attachments: dict[str, ValidatedAttachment],
|
||||
) -> dict[str, IntegratedStructureAttachment]:
|
||||
@@ -192,31 +236,50 @@ def import_integrated_structures(
|
||||
original_name: str,
|
||||
attachments: Iterable[BinaryIO],
|
||||
uploaded_by_id: int | None = None,
|
||||
) -> tuple[ParserSourceArtifact, dict]:
|
||||
"""Validate all inputs, stage all structures, then publish one transaction."""
|
||||
) -> tuple[ParserSourceArtifact | None, dict]:
|
||||
"""Validate all inputs and atomically publish known organizations."""
|
||||
workbook.seek(0)
|
||||
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
|
||||
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
|
||||
validated_attachments = _validated_attachments(attachments, parsed)
|
||||
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
|
||||
organizations = _resolve_organizations(parsed)
|
||||
resolution = _resolution_metadata(parsed, organizations)
|
||||
if not organizations:
|
||||
return None, {
|
||||
"status": "skipped",
|
||||
"reason": "no_known_organizations",
|
||||
"records_count": 0,
|
||||
**resolution,
|
||||
}
|
||||
existing = (
|
||||
ParserSourceArtifact.objects.filter(
|
||||
source=PARSER_SOURCE,
|
||||
status=ParserSourceArtifact.Status.PUBLISHED,
|
||||
metadata__composite_sha256=composite_hash,
|
||||
)
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
)
|
||||
if existing is not None:
|
||||
if (
|
||||
existing is not None
|
||||
and existing.metadata.get("composite_sha256") == composite_hash
|
||||
and existing.metadata.get("resolution_sha256")
|
||||
== resolution["resolution_sha256"]
|
||||
):
|
||||
return existing, {
|
||||
"status": "skipped",
|
||||
"reason": "identical_batch",
|
||||
"batch_id": existing.load_batch,
|
||||
"records_count": existing.published_count,
|
||||
"records_count": existing.metadata["batch_published_records_count"],
|
||||
**existing.metadata,
|
||||
}
|
||||
organizations = _resolve_organizations(parsed)
|
||||
known = _known_workbook(parsed, organizations)
|
||||
submitted_attachments_count = len(validated_attachments)
|
||||
validated_attachments = {
|
||||
key: value
|
||||
for key, value in validated_attachments.items()
|
||||
if key in known.documents
|
||||
}
|
||||
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
||||
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
|
||||
)
|
||||
@@ -230,20 +293,23 @@ def import_integrated_structures(
|
||||
parsed_count=len(parsed.structures),
|
||||
metadata={
|
||||
"composite_sha256": composite_hash,
|
||||
**resolution,
|
||||
"structures_count": parsed.counts[0],
|
||||
"event_rows_count": parsed.counts[1],
|
||||
"document_link_rows_count": parsed.counts[2],
|
||||
"unique_documents_count": parsed.counts[3],
|
||||
"events_without_documents_count": parsed.counts[4],
|
||||
"submitted_attachments_count": submitted_attachments_count,
|
||||
"imported_documents_count": len(known.documents),
|
||||
"attachments_count": len(validated_attachments),
|
||||
"missing_attachments_count": len(parsed.documents)
|
||||
"missing_attachments_count": len(known.documents)
|
||||
- len(validated_attachments),
|
||||
},
|
||||
)
|
||||
try:
|
||||
save_artifact_file(artifact, BytesIO(workbook_bytes))
|
||||
pending = []
|
||||
for index, (okpo, structure) in enumerate(parsed.structures.items(), 1):
|
||||
for index, (okpo, structure) in enumerate(known.structures.items(), 1):
|
||||
organization = organizations[okpo]
|
||||
external_id = f"integrated-structure:{okpo}"
|
||||
document_ids = {
|
||||
@@ -284,20 +350,29 @@ def import_integrated_structures(
|
||||
artifact.save(update_fields=["status", "updated_at"])
|
||||
|
||||
def finalize(artifact, result):
|
||||
_publish_relations(artifact, parsed, validated_attachments)
|
||||
_publish_relations(artifact, known, validated_attachments)
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.SUCCESS,
|
||||
records_count=result.published,
|
||||
records_count=len(organizations),
|
||||
)
|
||||
|
||||
result = publish_snapshot(
|
||||
artifact, record_source=RECORD_SOURCE, on_publish=finalize
|
||||
publish_snapshot(
|
||||
artifact,
|
||||
record_source=RECORD_SOURCE,
|
||||
on_publish=finalize,
|
||||
scope_organization_ids=[
|
||||
organization.uid for organization in organizations.values()
|
||||
]
|
||||
if resolution["skipped_organizations_count"]
|
||||
else None,
|
||||
)
|
||||
return artifact, {
|
||||
"status": "published",
|
||||
"status": "partial"
|
||||
if resolution["skipped_organizations_count"]
|
||||
else "published",
|
||||
"batch_id": batch_id,
|
||||
"records_count": result.published,
|
||||
"records_count": len(organizations),
|
||||
**artifact.metadata,
|
||||
}
|
||||
except Exception as exc:
|
||||
|
||||
@@ -3171,9 +3171,10 @@ class ParserUploadView(APIView):
|
||||
except Exception as exc:
|
||||
job.fail(error=type(exc).__name__)
|
||||
raise
|
||||
job.complete(result={**result, "artifact_id": str(artifact.uid)})
|
||||
artifact_id = str(artifact.uid) if artifact is not None else None
|
||||
job.complete(result={**result, "artifact_id": artifact_id})
|
||||
return api_response(
|
||||
{**result, "artifact_id": str(artifact.uid), "task_id": job.task_id},
|
||||
{**result, "artifact_id": artifact_id, "task_id": job.task_id},
|
||||
status_code=status.HTTP_200_OK,
|
||||
)
|
||||
if not descriptor.supports_file_upload:
|
||||
|
||||
Reference in New Issue
Block a user