Compare commits
2 Commits
hotfix/jso
...
hotfix/int
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
262193f121 | ||
| ecf3855aa7 |
74
docs/integrated-structures-import.md
Normal file
74
docs/integrated-structures-import.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Частичная загрузка интегрированных структур
|
||||||
|
|
||||||
|
`POST /api/v1/parsers/upload/integrated_structure_documents/` принимает multipart:
|
||||||
|
`workbook` (XLSX с листами `is_stages` и `is_stages_docs`) и необязательные
|
||||||
|
`attachments` (RTF). Требуются существующие права администратора загрузки.
|
||||||
|
|
||||||
|
Backend сначала проверяет весь файл и вложения. ОКПО сопоставляются только с
|
||||||
|
организациями канонического справочника (`directory_imported_at` заполнен).
|
||||||
|
Неизвестные ОКПО пропускаются; организации автоматически не создаются.
|
||||||
|
|
||||||
|
При наличии и известных, и неизвестных организаций HTTP-ответ остаётся 200,
|
||||||
|
`success=true`, `errors=null`. В `data` возвращаются:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "partial",
|
||||||
|
"records_count": 48,
|
||||||
|
"imported_organizations_count": 48,
|
||||||
|
"skipped_organizations_count": 2,
|
||||||
|
"skipped_organizations": [
|
||||||
|
{
|
||||||
|
"okpo": "02742757",
|
||||||
|
"code": "unknown_canonical_okpo",
|
||||||
|
"message": "Организация не загружена: ОКПО отсутствует в каноническом реестре."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"okpo": "10386058",
|
||||||
|
"code": "unknown_canonical_okpo",
|
||||||
|
"message": "Организация не загружена: ОКПО отсутствует в каноническом реестре."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"warnings": [
|
||||||
|
{
|
||||||
|
"code": "unknown_canonical_okpo",
|
||||||
|
"message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.",
|
||||||
|
"meta": {"okpos": ["02742757", "10386058"]}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Это фрагмент ответа: также возвращаются существующие `artifact_id`, `task_id`,
|
||||||
|
`batch_id` и статистика workbook. Тот же результат доступен в BackgroundJob.
|
||||||
|
Фронтенду следует показать количество загруженных организаций и все пропуски
|
||||||
|
из `skipped_organizations`, а не трактовать `success=true` как отсутствие предупреждений.
|
||||||
|
|
||||||
|
- Все ОКПО известны: `status=published`, список пропусков и warnings пусты.
|
||||||
|
- Все ОКПО неизвестны: HTTP 200, `status=skipped`, `reason=no_known_organizations`,
|
||||||
|
`records_count=0`, `artifact_id=null`, без `batch_id`; текущий snapshot не меняется.
|
||||||
|
- Идентичный последний опубликованный batch при неизменном сопоставлении с
|
||||||
|
реестром: `status=skipped`, `reason=identical_batch`; возвращается прежняя
|
||||||
|
статистика batch и его предупреждения, дублей нет.
|
||||||
|
- После пополнения канонического реестра тот же workbook можно загрузить повторно:
|
||||||
|
ранее пропущенные организации будут учтены.
|
||||||
|
- Два канонических совпадения одного ОКПО, повреждённый XLSX/RTF и нарушенные связи
|
||||||
|
остаются ошибками 422; известная часть при этих ошибках не публикуется.
|
||||||
|
|
||||||
|
`structures_count`, `event_rows_count`, `document_link_rows_count`,
|
||||||
|
`unique_documents_count`, `events_without_documents_count` описывают исходный
|
||||||
|
workbook. `records_count`/`imported_organizations_count` описывают загруженную часть
|
||||||
|
batch. `snapshot_records_count` описывает весь текущий snapshot, включая
|
||||||
|
сохранённые старые записи. `imported_documents_count`, `attachments_count` и
|
||||||
|
`missing_attachments_count` относятся к загруженным организациям;
|
||||||
|
`submitted_attachments_count` — ко всем переданным и проверенным вложениям.
|
||||||
|
Полностью пропущенный файл возвращает только счётчики организаций и предупреждения.
|
||||||
|
|
||||||
|
Частичная публикация атомарна и меняет данные только известных организаций из
|
||||||
|
файла. Прежние записи других организаций сохраняются. Документы и RTF,
|
||||||
|
связанные только с пропущенными организациями, не публикуются. Документ,
|
||||||
|
связанный также с известной организацией, включается в её данные.
|
||||||
|
Полная загрузка сохраняет прежнюю семантику замены snapshot.
|
||||||
|
|
||||||
|
Хотфикс относится к Mostovik dev. Он не добавляет этот источник на main и не
|
||||||
|
меняет другие источники или форматы JSON-экспорта.
|
||||||
@@ -117,11 +117,55 @@ def _resolve_organizations(parsed: ParsedWorkbook) -> dict[str, Organization]:
|
|||||||
if organization.okpo in by_okpo:
|
if organization.okpo in by_okpo:
|
||||||
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
|
raise IntegratedWorkbookError("ambiguous_canonical_okpo")
|
||||||
by_okpo[organization.okpo] = organization
|
by_okpo[organization.okpo] = organization
|
||||||
if len(by_okpo) != len(parsed.structures):
|
|
||||||
raise IntegratedWorkbookError("unknown_canonical_okpo")
|
|
||||||
return by_okpo
|
return by_okpo
|
||||||
|
|
||||||
|
|
||||||
|
def _known_workbook(
|
||||||
|
parsed: ParsedWorkbook, organizations: dict[str, Organization]
|
||||||
|
) -> ParsedWorkbook:
|
||||||
|
structures = {
|
||||||
|
okpo: item for okpo, item in parsed.structures.items() if okpo in organizations
|
||||||
|
}
|
||||||
|
events = [event for item in structures.values() for event in item.events]
|
||||||
|
document_ids = {document_id for event in events for document_id in event.documents}
|
||||||
|
return ParsedWorkbook(
|
||||||
|
structures=structures,
|
||||||
|
documents={
|
||||||
|
key: value for key, value in parsed.documents.items() if key in document_ids
|
||||||
|
},
|
||||||
|
links_count=sum(len(event.documents) for event in events),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolution_metadata(
|
||||||
|
parsed: ParsedWorkbook, organizations: dict[str, Organization]
|
||||||
|
) -> dict:
|
||||||
|
missing = sorted(set(parsed.structures) - organizations.keys())
|
||||||
|
message = "Организация не загружена: ОКПО отсутствует в каноническом реестре."
|
||||||
|
identity = "\n".join(
|
||||||
|
f"{okpo}:{organization.uid}"
|
||||||
|
for okpo, organization in sorted(organizations.items())
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"resolution_sha256": hashlib.sha256(identity.encode("utf-8")).hexdigest(),
|
||||||
|
"imported_organizations_count": len(organizations),
|
||||||
|
"skipped_organizations_count": len(missing),
|
||||||
|
"skipped_organizations": [
|
||||||
|
{"okpo": okpo, "code": "unknown_canonical_okpo", "message": message}
|
||||||
|
for okpo in missing
|
||||||
|
],
|
||||||
|
"warnings": [
|
||||||
|
{
|
||||||
|
"code": "unknown_canonical_okpo",
|
||||||
|
"message": "Часть организаций не загружена: ОКПО отсутствуют в каноническом реестре.",
|
||||||
|
"meta": {"okpos": missing},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
if missing
|
||||||
|
else [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _save_attachments(
|
def _save_attachments(
|
||||||
attachments: dict[str, ValidatedAttachment],
|
attachments: dict[str, ValidatedAttachment],
|
||||||
) -> dict[str, IntegratedStructureAttachment]:
|
) -> dict[str, IntegratedStructureAttachment]:
|
||||||
@@ -192,31 +236,50 @@ def import_integrated_structures(
|
|||||||
original_name: str,
|
original_name: str,
|
||||||
attachments: Iterable[BinaryIO],
|
attachments: Iterable[BinaryIO],
|
||||||
uploaded_by_id: int | None = None,
|
uploaded_by_id: int | None = None,
|
||||||
) -> tuple[ParserSourceArtifact, dict]:
|
) -> tuple[ParserSourceArtifact | None, dict]:
|
||||||
"""Validate all inputs, stage all structures, then publish one transaction."""
|
"""Validate all inputs and atomically publish known organizations."""
|
||||||
workbook.seek(0)
|
workbook.seek(0)
|
||||||
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
|
workbook_bytes = workbook.read(20 * 1024 * 1024 + 1)
|
||||||
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
|
parsed = parse_integrated_workbook(BytesIO(workbook_bytes))
|
||||||
validated_attachments = _validated_attachments(attachments, parsed)
|
validated_attachments = _validated_attachments(attachments, parsed)
|
||||||
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
|
composite_hash = _composite_hash(workbook_bytes, validated_attachments)
|
||||||
|
organizations = _resolve_organizations(parsed)
|
||||||
|
resolution = _resolution_metadata(parsed, organizations)
|
||||||
|
if not organizations:
|
||||||
|
return None, {
|
||||||
|
"status": "skipped",
|
||||||
|
"reason": "no_known_organizations",
|
||||||
|
"records_count": 0,
|
||||||
|
**resolution,
|
||||||
|
}
|
||||||
existing = (
|
existing = (
|
||||||
ParserSourceArtifact.objects.filter(
|
ParserSourceArtifact.objects.filter(
|
||||||
source=PARSER_SOURCE,
|
source=PARSER_SOURCE,
|
||||||
status=ParserSourceArtifact.Status.PUBLISHED,
|
status=ParserSourceArtifact.Status.PUBLISHED,
|
||||||
metadata__composite_sha256=composite_hash,
|
|
||||||
)
|
)
|
||||||
.order_by("-created_at")
|
.order_by("-created_at")
|
||||||
.first()
|
.first()
|
||||||
)
|
)
|
||||||
if existing is not None:
|
if (
|
||||||
|
existing is not None
|
||||||
|
and existing.metadata.get("composite_sha256") == composite_hash
|
||||||
|
and existing.metadata.get("resolution_sha256")
|
||||||
|
== resolution["resolution_sha256"]
|
||||||
|
):
|
||||||
return existing, {
|
return existing, {
|
||||||
"status": "skipped",
|
"status": "skipped",
|
||||||
"reason": "identical_batch",
|
"reason": "identical_batch",
|
||||||
"batch_id": existing.load_batch,
|
"batch_id": existing.load_batch,
|
||||||
"records_count": existing.published_count,
|
"records_count": existing.metadata["batch_published_records_count"],
|
||||||
**existing.metadata,
|
**existing.metadata,
|
||||||
}
|
}
|
||||||
organizations = _resolve_organizations(parsed)
|
known = _known_workbook(parsed, organizations)
|
||||||
|
submitted_attachments_count = len(validated_attachments)
|
||||||
|
validated_attachments = {
|
||||||
|
key: value
|
||||||
|
for key, value in validated_attachments.items()
|
||||||
|
if key in known.documents
|
||||||
|
}
|
||||||
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
load_log, batch_id = ParserLoadLogService.create_load_log_with_next_batch_id(
|
||||||
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
|
source=PARSER_SOURCE, status=ParserLoadLog.Status.IN_PROGRESS
|
||||||
)
|
)
|
||||||
@@ -230,20 +293,23 @@ def import_integrated_structures(
|
|||||||
parsed_count=len(parsed.structures),
|
parsed_count=len(parsed.structures),
|
||||||
metadata={
|
metadata={
|
||||||
"composite_sha256": composite_hash,
|
"composite_sha256": composite_hash,
|
||||||
|
**resolution,
|
||||||
"structures_count": parsed.counts[0],
|
"structures_count": parsed.counts[0],
|
||||||
"event_rows_count": parsed.counts[1],
|
"event_rows_count": parsed.counts[1],
|
||||||
"document_link_rows_count": parsed.counts[2],
|
"document_link_rows_count": parsed.counts[2],
|
||||||
"unique_documents_count": parsed.counts[3],
|
"unique_documents_count": parsed.counts[3],
|
||||||
"events_without_documents_count": parsed.counts[4],
|
"events_without_documents_count": parsed.counts[4],
|
||||||
|
"submitted_attachments_count": submitted_attachments_count,
|
||||||
|
"imported_documents_count": len(known.documents),
|
||||||
"attachments_count": len(validated_attachments),
|
"attachments_count": len(validated_attachments),
|
||||||
"missing_attachments_count": len(parsed.documents)
|
"missing_attachments_count": len(known.documents)
|
||||||
- len(validated_attachments),
|
- len(validated_attachments),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
save_artifact_file(artifact, BytesIO(workbook_bytes))
|
save_artifact_file(artifact, BytesIO(workbook_bytes))
|
||||||
pending = []
|
pending = []
|
||||||
for index, (okpo, structure) in enumerate(parsed.structures.items(), 1):
|
for index, (okpo, structure) in enumerate(known.structures.items(), 1):
|
||||||
organization = organizations[okpo]
|
organization = organizations[okpo]
|
||||||
external_id = f"integrated-structure:{okpo}"
|
external_id = f"integrated-structure:{okpo}"
|
||||||
document_ids = {
|
document_ids = {
|
||||||
@@ -284,20 +350,29 @@ def import_integrated_structures(
|
|||||||
artifact.save(update_fields=["status", "updated_at"])
|
artifact.save(update_fields=["status", "updated_at"])
|
||||||
|
|
||||||
def finalize(artifact, result):
|
def finalize(artifact, result):
|
||||||
_publish_relations(artifact, parsed, validated_attachments)
|
_publish_relations(artifact, known, validated_attachments)
|
||||||
ParserLoadLogService.update(
|
ParserLoadLogService.update(
|
||||||
load_log,
|
load_log,
|
||||||
status=ParserLoadLog.Status.SUCCESS,
|
status=ParserLoadLog.Status.SUCCESS,
|
||||||
records_count=result.published,
|
records_count=len(organizations),
|
||||||
)
|
)
|
||||||
|
|
||||||
result = publish_snapshot(
|
publish_snapshot(
|
||||||
artifact, record_source=RECORD_SOURCE, on_publish=finalize
|
artifact,
|
||||||
|
record_source=RECORD_SOURCE,
|
||||||
|
on_publish=finalize,
|
||||||
|
scope_organization_ids=[
|
||||||
|
organization.uid for organization in organizations.values()
|
||||||
|
]
|
||||||
|
if resolution["skipped_organizations_count"]
|
||||||
|
else None,
|
||||||
)
|
)
|
||||||
return artifact, {
|
return artifact, {
|
||||||
"status": "published",
|
"status": "partial"
|
||||||
|
if resolution["skipped_organizations_count"]
|
||||||
|
else "published",
|
||||||
"batch_id": batch_id,
|
"batch_id": batch_id,
|
||||||
"records_count": result.published,
|
"records_count": len(organizations),
|
||||||
**artifact.metadata,
|
**artifact.metadata,
|
||||||
}
|
}
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
@@ -3171,9 +3171,10 @@ class ParserUploadView(APIView):
|
|||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
job.fail(error=type(exc).__name__)
|
job.fail(error=type(exc).__name__)
|
||||||
raise
|
raise
|
||||||
job.complete(result={**result, "artifact_id": str(artifact.uid)})
|
artifact_id = str(artifact.uid) if artifact is not None else None
|
||||||
|
job.complete(result={**result, "artifact_id": artifact_id})
|
||||||
return api_response(
|
return api_response(
|
||||||
{**result, "artifact_id": str(artifact.uid), "task_id": job.task_id},
|
{**result, "artifact_id": artifact_id, "task_id": job.task_id},
|
||||||
status_code=status.HTTP_200_OK,
|
status_code=status.HTTP_200_OK,
|
||||||
)
|
)
|
||||||
if not descriptor.supports_file_upload:
|
if not descriptor.supports_file_upload:
|
||||||
|
|||||||
273
tests/apps/parsers/test_integrated_structure_partial_import.py
Normal file
273
tests/apps/parsers/test_integrated_structure_partial_import.py
Normal file
@@ -0,0 +1,273 @@
|
|||||||
|
"""Partial publication of valid workbooks with organizations outside the directory."""
|
||||||
|
from io import BytesIO
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||||
|
from django.test import TestCase, override_settings
|
||||||
|
from django.utils import timezone
|
||||||
|
from openpyxl import load_workbook
|
||||||
|
from organizations.models import Organization, OrganizationSourceRecord
|
||||||
|
from rest_framework.test import APIClient
|
||||||
|
|
||||||
|
from tests.apps.parsers.test_integrated_structure_workbook import workbook_bytes
|
||||||
|
from tests.apps.user.factories import UserFactory
|
||||||
|
|
||||||
|
|
||||||
|
def multi_organization_workbook(*okpos):
|
||||||
|
combined = load_workbook(BytesIO(workbook_bytes(okpo=okpos[0])))
|
||||||
|
for index, okpo in enumerate(okpos[1:], 1):
|
||||||
|
extra = load_workbook(BytesIO(workbook_bytes(okpo=okpo, doc_id=93 + index)))
|
||||||
|
for name in combined.sheetnames:
|
||||||
|
for row in extra[name].iter_rows(min_row=2, values_only=True):
|
||||||
|
combined[name].append(row)
|
||||||
|
extra.close()
|
||||||
|
stream = BytesIO()
|
||||||
|
combined.save(stream)
|
||||||
|
combined.close()
|
||||||
|
return stream.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
class IntegratedStructurePartialImportTest(TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
|
||||||
|
self.private = TemporaryDirectory()
|
||||||
|
self.addCleanup(self.private.cleanup)
|
||||||
|
self.settings = override_settings(
|
||||||
|
PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name,
|
||||||
|
MEDIA_ROOT=self.private.name + "/public",
|
||||||
|
)
|
||||||
|
self.settings.enable()
|
||||||
|
self.addCleanup(self.settings.disable)
|
||||||
|
Organization.objects.create(
|
||||||
|
name="АО Тест",
|
||||||
|
inn="1234567890",
|
||||||
|
ogrn="1027700132195",
|
||||||
|
okpo="00084008",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _upload_workbook(self, raw, attachments=()):
|
||||||
|
client = APIClient()
|
||||||
|
client.force_authenticate(UserFactory.create_superuser())
|
||||||
|
return client.post(
|
||||||
|
"/api/v1/parsers/upload/integrated_structure_documents/",
|
||||||
|
{
|
||||||
|
"workbook": SimpleUploadedFile("test.xlsx", raw),
|
||||||
|
"attachments": list(attachments),
|
||||||
|
},
|
||||||
|
format="multipart",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_partial_upload_reports_unknown_okpo_and_publishes_known_relations(self):
|
||||||
|
from apps.core.models import BackgroundJob
|
||||||
|
from apps.parsers.models import (
|
||||||
|
IntegratedStructureDocument,
|
||||||
|
IntegratedStructureEvent,
|
||||||
|
)
|
||||||
|
|
||||||
|
raw = multi_organization_workbook("00084008", "02742757")
|
||||||
|
response = self._upload_workbook(raw)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.data["data"]
|
||||||
|
self.assertEqual(data["status"], "partial")
|
||||||
|
self.assertEqual(data["records_count"], 1)
|
||||||
|
self.assertEqual(data["imported_organizations_count"], 1)
|
||||||
|
self.assertEqual(data["skipped_organizations_count"], 1)
|
||||||
|
self.assertEqual(
|
||||||
|
data["skipped_organizations"],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"okpo": "02742757",
|
||||||
|
"code": "unknown_canonical_okpo",
|
||||||
|
"message": "Организация не загружена: ОКПО отсутствует в каноническом реестре.",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
)
|
||||||
|
self.assertEqual(data["warnings"][0]["meta"]["okpos"], ["02742757"])
|
||||||
|
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
|
||||||
|
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
|
||||||
|
self.assertEqual(
|
||||||
|
BackgroundJob.objects.get().result["skipped_organizations"],
|
||||||
|
data["skipped_organizations"],
|
||||||
|
)
|
||||||
|
second = self._upload_workbook(raw).data["data"]
|
||||||
|
self.assertEqual(second["reason"], "identical_batch")
|
||||||
|
self.assertEqual(second["skipped_organizations"], data["skipped_organizations"])
|
||||||
|
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
|
||||||
|
|
||||||
|
def test_all_unknown_upload_preserves_existing_snapshot(self):
|
||||||
|
from apps.parsers.models import ParserSourceArtifact
|
||||||
|
|
||||||
|
self.assertEqual(self._upload_workbook(workbook_bytes()).status_code, 200)
|
||||||
|
artifact_ids = list(ParserSourceArtifact.objects.values_list("uid", flat=True))
|
||||||
|
response = self._upload_workbook(workbook_bytes(okpo="02742757"))
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
data = response.data["data"]
|
||||||
|
self.assertEqual(data["status"], "skipped")
|
||||||
|
self.assertEqual(data["reason"], "no_known_organizations")
|
||||||
|
self.assertEqual(data["records_count"], 0)
|
||||||
|
self.assertEqual(data["imported_organizations_count"], 0)
|
||||||
|
self.assertEqual(data["skipped_organizations"][0]["okpo"], "02742757")
|
||||||
|
self.assertIsNone(data["artifact_id"])
|
||||||
|
self.assertEqual(
|
||||||
|
list(ParserSourceArtifact.objects.values_list("uid", flat=True)),
|
||||||
|
artifact_ids,
|
||||||
|
)
|
||||||
|
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
|
||||||
|
|
||||||
|
def test_partial_upload_preserves_previously_published_unknown_company(self):
|
||||||
|
other = Organization.objects.create(
|
||||||
|
name="Other",
|
||||||
|
inn="1234567891",
|
||||||
|
ogrn="1027700132196",
|
||||||
|
okpo="02742757",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
raw = multi_organization_workbook("00084008", "02742757")
|
||||||
|
self.assertEqual(self._upload_workbook(raw).status_code, 200)
|
||||||
|
record = OrganizationSourceRecord.objects.get(extension__organization=other)
|
||||||
|
event_ids = list(record.integrated_events.values_list("id", flat=True))
|
||||||
|
other.directory_imported_at = None
|
||||||
|
other.save(update_fields=["directory_imported_at"])
|
||||||
|
response = self._upload_workbook(raw)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response.data["data"]["status"], "partial")
|
||||||
|
self.assertEqual(response.data["data"]["records_count"], 1)
|
||||||
|
self.assertEqual(
|
||||||
|
list(record.integrated_events.values_list("id", flat=True)), event_ids
|
||||||
|
)
|
||||||
|
self.assertEqual(OrganizationSourceRecord.objects.count(), 2)
|
||||||
|
|
||||||
|
def test_repeat_after_directory_update_imports_previously_skipped_company(self):
|
||||||
|
from apps.parsers.models import IntegratedStructureEvent
|
||||||
|
|
||||||
|
raw = multi_organization_workbook("00084008", "02742757")
|
||||||
|
self.assertEqual(self._upload_workbook(raw).status_code, 200)
|
||||||
|
Organization.objects.create(
|
||||||
|
name="Other",
|
||||||
|
inn="1234567891",
|
||||||
|
ogrn="1027700132196",
|
||||||
|
okpo="02742757",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
response = self._upload_workbook(raw)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response.data["data"]["status"], "published")
|
||||||
|
self.assertEqual(response.data["data"]["records_count"], 2)
|
||||||
|
self.assertEqual(response.data["data"]["skipped_organizations"], [])
|
||||||
|
self.assertEqual(IntegratedStructureEvent.objects.count(), 4)
|
||||||
|
|
||||||
|
def test_reimport_older_workbook_is_not_skipped_after_another_publication(self):
|
||||||
|
first = workbook_bytes(doc_id=93)
|
||||||
|
self.assertEqual(self._upload_workbook(first).status_code, 200)
|
||||||
|
self.assertEqual(
|
||||||
|
self._upload_workbook(workbook_bytes(doc_id=94)).status_code, 200
|
||||||
|
)
|
||||||
|
response = self._upload_workbook(first)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response.data["data"]["status"], "published")
|
||||||
|
record = OrganizationSourceRecord.objects.get()
|
||||||
|
self.assertEqual(
|
||||||
|
record.integrated_events.get(external_id="1137")
|
||||||
|
.document_links.get()
|
||||||
|
.document.external_id,
|
||||||
|
"93",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_ambiguous_canonical_okpo_still_rejects_entire_upload(self):
|
||||||
|
Organization.objects.create(
|
||||||
|
name="Ambiguous",
|
||||||
|
inn="1234567891",
|
||||||
|
ogrn="1027700132196",
|
||||||
|
okpo="00084008",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
response = self._upload_workbook(
|
||||||
|
multi_organization_workbook("00084008", "02742757")
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 422)
|
||||||
|
self.assertEqual(response.data["errors"][0]["code"], "ambiguous_canonical_okpo")
|
||||||
|
self.assertEqual(OrganizationSourceRecord.objects.count(), 0)
|
||||||
|
|
||||||
|
def test_attachments_for_unknown_organizations_are_not_published(self):
|
||||||
|
from apps.parsers.models import (
|
||||||
|
IntegratedStructureAttachment,
|
||||||
|
IntegratedStructureDocument,
|
||||||
|
)
|
||||||
|
|
||||||
|
response = self._upload_workbook(
|
||||||
|
multi_organization_workbook("00084008", "02742757"),
|
||||||
|
[SimpleUploadedFile("000094.rtf", b"{\\rtf1\\ansi omitted}")],
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(IntegratedStructureAttachment.objects.count(), 0)
|
||||||
|
self.assertEqual(
|
||||||
|
list(
|
||||||
|
IntegratedStructureDocument.objects.values_list(
|
||||||
|
"external_id", flat=True
|
||||||
|
)
|
||||||
|
),
|
||||||
|
["93"],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_partial_upload_retains_company_absent_from_current_workbook(self):
|
||||||
|
other = Organization.objects.create(
|
||||||
|
name="Existing",
|
||||||
|
inn="1234567891",
|
||||||
|
ogrn="1027700132196",
|
||||||
|
okpo="10386058",
|
||||||
|
directory_imported_at=timezone.now(),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self._upload_workbook(
|
||||||
|
multi_organization_workbook("00084008", "10386058")
|
||||||
|
).status_code,
|
||||||
|
200,
|
||||||
|
)
|
||||||
|
previous = OrganizationSourceRecord.objects.get(extension__organization=other)
|
||||||
|
response = self._upload_workbook(
|
||||||
|
multi_organization_workbook("00084008", "02742757")
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertEqual(response.data["data"]["records_count"], 1)
|
||||||
|
self.assertTrue(
|
||||||
|
OrganizationSourceRecord.objects.filter(pk=previous.pk).exists()
|
||||||
|
)
|
||||||
|
self.assertEqual(previous.integrated_events.count(), 2)
|
||||||
|
|
||||||
|
def test_failed_partial_publication_keeps_previous_relations(self):
|
||||||
|
from apps.parsers.integrated_structures import import_integrated_structures
|
||||||
|
from apps.parsers.models import (
|
||||||
|
IntegratedStructureDocument,
|
||||||
|
IntegratedStructureEvent,
|
||||||
|
)
|
||||||
|
|
||||||
|
import_integrated_structures(
|
||||||
|
workbook=BytesIO(workbook_bytes()), original_name="old.xlsx", attachments=[]
|
||||||
|
)
|
||||||
|
previous_events = list(
|
||||||
|
IntegratedStructureEvent.objects.values_list("id", flat=True)
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"apps.parsers.integrated_structures._publish_relations",
|
||||||
|
side_effect=RuntimeError("storage_failure"),
|
||||||
|
), self.assertRaisesRegex(RuntimeError, "storage_failure"):
|
||||||
|
import_integrated_structures(
|
||||||
|
workbook=BytesIO(multi_organization_workbook("00084008", "02742757")),
|
||||||
|
original_name="partial.xlsx",
|
||||||
|
attachments=[],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
list(IntegratedStructureEvent.objects.values_list("id", flat=True)),
|
||||||
|
previous_events,
|
||||||
|
)
|
||||||
|
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
|
||||||
|
|
||||||
|
def test_malformed_unknown_rows_still_reject_entire_workbook(self):
|
||||||
|
response = self._upload_workbook(
|
||||||
|
workbook_bytes(okpo="02742757", document_stage=9999)
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 422)
|
||||||
|
self.assertEqual(response.data["errors"][0]["code"], "unknown_stage")
|
||||||
|
self.assertEqual(OrganizationSourceRecord.objects.count(), 0)
|
||||||
Reference in New Issue
Block a user