feat: add ROPK sanctions file source
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 1m10s
CI/CD Pipeline / Build and Push Images (push) Successful in 18s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 31s

This commit is contained in:
2026-08-20 15:50:04 +02:00
parent f2e36a8991
commit 7c50cf82ad
23 changed files with 1520 additions and 41 deletions

View File

@@ -22,6 +22,7 @@ from organizations.models import (
OrganizationSourceFinancialLine,
OrganizationSourceRecord,
PlannedInspectionExtension,
SanctionsExtension,
SourceGroup,
)
from organizations.source_record_export import (
@@ -92,9 +93,93 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
call_command("build_source_record_exports", stdout=command_output)
generation = load_current_source_record_export_generation()
self.assertEqual(generation.artifacts_count, 31)
self.assertEqual(generation.artifacts_count, 34)
self.assertEqual(generation.export_year, timezone.localdate().year)
self.assertIn('"artifacts_count": 31', command_output.getvalue())
self.assertIn('"artifacts_count": 34', command_output.getvalue())
def test_sanctions_xlsx_export_uses_source_contract_columns_and_boolean_flags(self):
organization = Organization.objects.create(
name="АО Экспорт Санкции",
inn="0012345678",
ogrn="1027700132195",
okpo="00123456",
)
extension = SanctionsExtension.objects.create(
organization=organization,
title="РОПК — Санкции",
)
OrganizationSourceRecord.objects.create(
extension=extension,
source="ropk_sanctions",
record_type="organization_sanctions",
external_id="000001",
title=organization.name,
status="active",
payload={
"rn": "000001",
"uk_hm_treasury": True,
"european_union": False,
"united_states": True,
"switzerland": False,
"united_states_sectoral": True,
"uk_uksl": False,
"ukraine": True,
},
)
build_source_record_export_artifacts()
self.client.force_authenticate(UserFactory.create_superuser())
response = self.client.post(
self.url,
{"sources": [SourceGroup.SANCTIONS.value], "format": "xlsx"},
format="json",
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
with zipfile.ZipFile(BytesIO(self._response_body(response))) as archive:
self.assertEqual(archive.namelist(), ["ropk-sanctions.xlsx"])
workbook = load_workbook(
BytesIO(archive.read("ropk-sanctions.xlsx")),
read_only=True,
data_only=True,
)
rows = list(workbook["data"].iter_rows(values_only=True))
workbook.close()
self.assertEqual(
rows[0],
(
"Наименование",
"rn",
"ogrn",
"inn",
"okpo",
"Санкции - Великобритания HM Treasury",
"Санкции - Евросоюз",
"Санкции - США",
"Санкции - Швейцария",
"Санкции секторальные - США",
"Санкции - Великобритания UKSL",
"Санкции - Украина",
),
)
self.assertEqual(
rows[1],
(
"АО Экспорт Санкции",
"000001",
"1027700132195",
"0012345678",
"00123456",
1,
0,
1,
0,
1,
0,
1,
),
)
def test_generation_contains_only_records_from_its_calendar_year(self):
export_year = 2026
@@ -313,7 +398,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
self.assertEqual(
response["X-Source-Export-Generated-At"], generation.generated_at
)
self.assertEqual(generation.artifacts_count, 31)
self.assertEqual(generation.artifacts_count, 34)
self.assertIn(
'filename="planned-inspections__financial-indicators_',
response["Content-Disposition"],
@@ -508,8 +593,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
key=lambda item: item.file_name,
)
self.assertEqual(generation.artifacts_count, 31)
self.assertEqual(generation.files_count, 32)
self.assertEqual(generation.artifacts_count, 34)
self.assertEqual(generation.files_count, 35)
self.assertEqual(
[item.file_name for item in artifacts],
[
@@ -707,7 +792,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
first_generation = build_source_record_export_artifacts()
current_generation = load_current_source_record_export_generation()
self.assertEqual(first_generation.artifacts_count, 31)
self.assertEqual(first_generation.artifacts_count, 34)
self.assertEqual(
current_generation.generation_id, first_generation.generation_id
)

View File

@@ -145,7 +145,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase):
result = refresh_source_record_export_artifacts()
self.assertEqual(result["status"], "success")
self.assertEqual(result["artifacts_count"], 31)
self.assertEqual(result["artifacts_count"], 34)
self.assertEqual(result["export_year"], timezone.localdate().year)
self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))

View File

@@ -64,7 +64,11 @@ class TestCompaniesCommandsTest(TestCase):
)
expected_groups = {choice.value for choice in SourceGroup}
expected_sources = {choice.value for choice in ParserLoadLog.Source}
expected_sources = {
choice.value
for choice in ParserLoadLog.Source
if choice is not ParserLoadLog.Source.ROPK_SANCTIONS
}
for company in companies:
self.assertEqual(
set(company.source_extensions.values_list("source_group", flat=True)),

View File

@@ -0,0 +1,500 @@
from __future__ import annotations
from io import BytesIO
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from unittest.mock import patch
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
from apps.parsers.ropk_sanctions import (
ROPK_SANCTIONS_MAX_BYTES,
ROPK_SANCTIONS_MIME,
ROPK_SANCTIONS_RECORD_TYPE,
ROPK_SANCTIONS_SOURCE,
RopkSanctionsValidationError,
import_ropk_sanctions,
)
from apps.parsers.tasks import parse_ropk_sanctions
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase
from django.urls import reverse
from django.utils import timezone
from openpyxl import Workbook
from organizations.models import (
Organization,
OrganizationSourceRecord,
SanctionsExtension,
)
from rest_framework import status
from rest_framework.test import APITestCase
from tests.apps.user.factories import UserFactory
HEADERS = [
"rn",
"ogrn",
"inn",
"okpo",
"Санкции - Великобритания HM Treasury",
"Санкции - Евросоюз",
"Санкции - США",
"Санкции - Швейцария",
"Санкции секторальные - США",
"Санкции - Великобритания UKSL",
"Санкции - Украина",
]
def _workbook(rows: list[list[object]], *, headers: list[str] | None = None) -> BytesIO:
workbook = Workbook()
sheet = workbook.active
sheet.append(headers or HEADERS)
for row in rows:
sheet.append(row)
output = BytesIO()
workbook.save(output)
workbook.close()
output.seek(0)
return output
def _row(
rn: str,
*,
ogrn: str = "1027700132195",
inn: str = "0012345678",
okpo: str = "00123456",
flags: tuple[object, ...] = (1, 0, 1, 0, 1, 0, 1),
) -> list[object]:
return [rn, ogrn, inn, okpo, *flags]
class RopkSanctionsImportTest(TestCase):
def setUp(self):
self.organization = Organization.objects.create(
name="АО Санкции",
inn="0012345678",
ogrn="1027700132195",
okpo="00123456",
opk_registry_membership=True,
directory_imported_at=timezone.now(),
)
def test_import_enriches_by_identifiers_and_publishes_typed_payload(self):
artifact, result = import_ropk_sanctions(
handle=_workbook([_row("000001")]),
original_name="sanctions.xlsx",
load_batch=1,
uploaded_by_id=None,
)
self.assertFalse(result.skipped)
self.assertEqual(result.parsed, 1)
self.assertEqual(result.published, 1)
self.assertEqual(result.quarantined, 0)
self.assertEqual(artifact.status, ParserSourceArtifact.Status.PUBLISHED)
record = OrganizationSourceRecord.objects.get()
self.assertEqual(record.source, ROPK_SANCTIONS_SOURCE)
self.assertEqual(record.record_type, ROPK_SANCTIONS_RECORD_TYPE)
self.assertEqual(record.external_id, "000001")
self.assertEqual(record.title, self.organization.name)
self.assertEqual(record.record_date, "")
self.assertEqual(
record.payload,
{
"rn": "000001",
"uk_hm_treasury": True,
"european_union": False,
"united_states": True,
"switzerland": False,
"united_states_sectoral": True,
"uk_uksl": False,
"ukraine": True,
},
)
self.assertIsInstance(record.payload["uk_hm_treasury"], bool)
self.assertEqual(record.extension.organization_id, self.organization.uid)
def test_import_enriches_missing_inn_and_ogrn_by_unique_okpo(self):
_, result = import_ropk_sanctions(
handle=_workbook([_row("2", ogrn="", inn="")]),
original_name="sanctions.xlsx",
load_batch=2,
uploaded_by_id=None,
)
self.assertEqual(result.published, 1)
record = OrganizationSourceRecord.objects.get()
self.assertEqual(record.extension.organization.inn, "0012345678")
self.assertEqual(record.extension.organization.ogrn, "1027700132195")
def test_import_preserves_full_fourteen_digit_okpo(self):
self.organization.okpo = "00123456789012"
self.organization.save(update_fields=["okpo"])
_, result = import_ropk_sanctions(
handle=_workbook([_row("000014", ogrn="", inn="", okpo="00123456789012")]),
original_name="sanctions.xlsx",
load_batch=21,
uploaded_by_id=None,
)
self.assertEqual(result.published, 1)
record = OrganizationSourceRecord.objects.get()
self.assertEqual(record.extension.organization.okpo, "00123456789012")
self.assertEqual(record.external_id, "000014")
def test_invalid_rows_are_quarantined_without_partial_records(self):
_, result = import_ropk_sanctions(
handle=_workbook(
[
_row("duplicate"),
_row("duplicate"),
_row("bad-flag", flags=(2, 0, 0, 0, 0, 0, 0)),
_row("mismatch", okpo="99999999"),
]
),
original_name="sanctions.xlsx",
load_batch=3,
uploaded_by_id=None,
)
self.assertEqual(result.published, 0)
self.assertEqual(result.quarantined, 4)
self.assertEqual(
result.reasons,
{"duplicate_rn": 2, "invalid_flag": 1, "identifier_mismatch": 1},
)
self.assertFalse(OrganizationSourceRecord.objects.exists())
self.assertEqual(
ParserStagedRecord.objects.filter(
disposition=ParserStagedRecord.Disposition.QUARANTINED
).count(),
4,
)
def test_new_snapshot_atomically_replaces_previous_records(self):
import_ropk_sanctions(
handle=_workbook([_row("old")]),
original_name="old.xlsx",
load_batch=4,
uploaded_by_id=None,
)
import_ropk_sanctions(
handle=_workbook([_row("new")]),
original_name="new.xlsx",
load_batch=5,
uploaded_by_id=None,
)
self.assertEqual(
list(
OrganizationSourceRecord.objects.filter(
source=ROPK_SANCTIONS_SOURCE
).values_list("external_id", flat=True)
),
["new"],
)
def test_duplicate_checksum_is_skipped_without_changing_snapshot(self):
raw = _workbook([_row("same")]).getvalue()
_, first = import_ropk_sanctions(
handle=BytesIO(raw),
original_name="first.xlsx",
load_batch=6,
uploaded_by_id=None,
)
artifact, second = import_ropk_sanctions(
handle=BytesIO(raw),
original_name="second.xlsx",
load_batch=7,
uploaded_by_id=None,
)
self.assertFalse(first.skipped)
self.assertTrue(second.skipped)
self.assertEqual(artifact.status, ParserSourceArtifact.Status.SKIPPED)
self.assertEqual(
OrganizationSourceRecord.objects.filter(
source=ROPK_SANCTIONS_SOURCE
).count(),
1,
)
def test_invalid_header_order_rejects_batch_and_preserves_snapshot(self):
import_ropk_sanctions(
handle=_workbook([_row("old")]),
original_name="old.xlsx",
load_batch=8,
uploaded_by_id=None,
)
reordered = [HEADERS[1], HEADERS[0], *HEADERS[2:]]
with self.assertRaisesMessage(RopkSanctionsValidationError, "invalid_headers"):
import_ropk_sanctions(
handle=_workbook([_row("new")], headers=reordered),
original_name="invalid.xlsx",
load_batch=9,
uploaded_by_id=None,
)
self.assertEqual(
list(
OrganizationSourceRecord.objects.filter(
source=ROPK_SANCTIONS_SOURCE
).values_list("external_id", flat=True)
),
["old"],
)
self.assertEqual(
ParserSourceArtifact.objects.filter(
source=ROPK_SANCTIONS_SOURCE,
status=ParserSourceArtifact.Status.REJECTED,
).count(),
1,
)
def test_archive_expansion_limit_rejects_file_but_retains_raw_artifact(self):
with patch(
"apps.parsers.ropk_sanctions.ROPK_SANCTIONS_MAX_UNCOMPRESSED_BYTES",
1,
), self.assertRaisesMessage(
RopkSanctionsValidationError,
"xlsx_uncompressed_size_exceeded",
):
import_ropk_sanctions(
handle=_workbook([_row("zip-limit")]),
original_name="sanctions.xlsx",
load_batch=22,
uploaded_by_id=None,
)
artifact = ParserSourceArtifact.objects.get(source=ROPK_SANCTIONS_SOURCE)
self.assertEqual(artifact.status, ParserSourceArtifact.Status.REJECTED)
self.assertTrue(artifact.file.name)
self.assertTrue(artifact.file.storage.exists(artifact.file.name))
class RopkSanctionsApiTest(APITestCase):
def setUp(self):
self.user = UserFactory.create_user()
self.admin = UserFactory.create_user(is_staff=True)
self.upload_url = reverse(
"api_v1:parsers:upload-parser-data",
args=[ROPK_SANCTIONS_SOURCE],
)
def _upload(
self, name: str, content: bytes, content_type: str = ROPK_SANCTIONS_MIME
):
return self.client.post(
self.upload_url,
{"file": SimpleUploadedFile(name, content, content_type=content_type)},
format="multipart",
)
def test_upload_is_admin_only_and_queues_new_source(self):
self.client.force_authenticate(self.user)
self.assertEqual(
self._upload("sanctions.xlsx", b"xlsx").status_code,
status.HTTP_403_FORBIDDEN,
)
self.client.force_authenticate(self.admin)
with patch(
"apps.parsers.views._save_uploaded_parser_file",
return_value="parser_uploads/sanctions.xlsx",
), patch(
"apps.parsers.tasks.parse_ropk_sanctions.apply_async",
side_effect=lambda **kwargs: SimpleNamespace(id=kwargs["task_id"]),
):
response = self._upload("sanctions.xlsx", b"xlsx")
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
self.assertEqual(response.data["data"]["source"], ROPK_SANCTIONS_SOURCE)
self.assertEqual(response.data["data"]["status"], JobStatus.PENDING)
self.assertEqual(
response.data["data"]["task_ids"],
[response.data["data"]["task_id"]],
)
def test_upload_rejects_extension_mime_and_size_before_queueing(self):
self.client.force_authenticate(self.admin)
with patch("apps.parsers.views._save_uploaded_parser_file") as save_file, patch(
"apps.parsers.tasks.parse_ropk_sanctions.apply_async"
) as apply_async:
extension = self._upload("sanctions.csv", b"csv", "text/csv")
mime = self._upload("sanctions.xlsx", b"xlsx", "application/zip")
with patch("apps.parsers.views.ROPK_SANCTIONS_MAX_BYTES", 1):
oversized = self._upload("sanctions.xlsx", b"xx")
self.assertEqual(extension.status_code, status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
self.assertEqual(extension.data["errors"][0]["code"], "unsupported_media_type")
self.assertEqual(mime.status_code, status.HTTP_415_UNSUPPORTED_MEDIA_TYPE)
self.assertEqual(mime.data["errors"][0]["code"], "unsupported_media_type")
self.assertEqual(
oversized.status_code, status.HTTP_413_REQUEST_ENTITY_TOO_LARGE
)
self.assertEqual(oversized.data["errors"][0]["code"], "file_too_large")
save_file.assert_not_called()
apply_async.assert_not_called()
self.assertEqual(ROPK_SANCTIONS_MAX_BYTES, 25 * 1024 * 1024)
def test_upload_requires_file_before_creating_job(self):
self.client.force_authenticate(self.admin)
response = self.client.post(self.upload_url, {}, format="multipart")
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
self.assertEqual(response.data["errors"][0]["code"], "file_required")
self.assertFalse(BackgroundJob.objects.exists())
def test_upload_rejects_concurrent_job_before_saving_file(self):
BackgroundJob.objects.create(
task_id="ropk-running",
task_name="apps.parsers.tasks.parse_ropk_sanctions",
status=JobStatus.STARTED,
)
self.client.force_authenticate(self.admin)
with patch("apps.parsers.views._save_uploaded_parser_file") as save_file:
response = self._upload("sanctions.xlsx", b"xlsx")
self.assertEqual(response.status_code, status.HTTP_409_CONFLICT)
self.assertEqual(response.data["errors"][0]["code"], "upload_already_running")
save_file.assert_not_called()
def test_catalog_records_and_file_only_contract_are_available(self):
organization = Organization.objects.create(
name="АО API Санкции",
inn="7707083801",
ogrn="1027700132101",
okpo="12345671",
opk_registry_membership=True,
directory_imported_at=timezone.now(),
)
extension = SanctionsExtension.objects.create(
organization=organization,
title="РОПК — Санкции",
)
OrganizationSourceRecord.objects.create(
extension=extension,
source=ROPK_SANCTIONS_SOURCE,
record_type=ROPK_SANCTIONS_RECORD_TYPE,
external_id="10",
title=organization.name,
status="active",
payload={
"rn": "10",
"uk_hm_treasury": True,
"european_union": False,
"united_states": False,
"switzerland": False,
"united_states_sectoral": False,
"uk_uksl": False,
"ukraine": False,
},
)
self.client.force_authenticate(self.user)
card = self.client.get(
reverse(
"api_v1:sources:source-cards-detail",
kwargs={"slug": "ropk-sanctions"},
)
)
records = self.client.get(
reverse("api_v2:organizations:organization-source-records-list"),
{
"source_group": "sanctions",
"source": ROPK_SANCTIONS_SOURCE,
"record_type": ROPK_SANCTIONS_RECORD_TYPE,
"ordering": "payload__uk_hm_treasury",
},
)
self.client.force_authenticate(self.admin)
run = self.client.post(
reverse(
"api_v1:parsers:run-parser",
args=[ROPK_SANCTIONS_SOURCE],
),
{},
format="json",
)
self.assertEqual(card.status_code, status.HTTP_200_OK)
self.assertEqual(card.data["data"]["slug"], "ropk-sanctions")
self.assertFalse(card.data["data"]["supports_refresh"])
self.assertEqual(card.data["data"]["upload_url"], self.upload_url)
self.assertEqual(records.status_code, status.HTTP_200_OK)
self.assertEqual(records.data["meta"]["pagination"]["total_count"], 1)
self.assertEqual(records.data["data"][0]["payload"]["rn"], "10")
self.assertIs(records.data["data"][0]["payload"]["uk_hm_treasury"], True)
self.assertEqual(run.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
self.assertEqual(run.data["errors"][0]["code"], "parser_run_not_supported")
refresh = self.client.post(
reverse(
"api_v1:sources:source-cards-refresh",
kwargs={"slug": "ropk-sanctions"},
),
{},
format="json",
)
self.assertEqual(refresh.status_code, status.HTTP_405_METHOD_NOT_ALLOWED)
self.assertEqual(refresh.data["errors"][0]["code"], "refresh_not_supported")
def test_openapi_exposes_source_specific_upload_and_boolean_payload(self):
response = self.client.get(
reverse("schema-swagger-ui"),
{"format": "openapi"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
schema = response.content.decode("utf-8")
self.assertIn("/api/v1/parsers/upload/ropk_sanctions/", schema)
self.assertIn("XLSX-файл РОПК — Санкции", schema)
self.assertIn("uk_hm_treasury", schema)
self.assertIn("european_union", schema)
class RopkSanctionsTaskTest(TestCase):
def test_task_completes_job_and_reports_contract_counts(self):
Organization.objects.create(
name="АО Задача Санкции",
inn="0012345678",
ogrn="1027700132195",
okpo="00123456",
opk_registry_membership=True,
directory_imported_at=timezone.now(),
)
workbook = _workbook([_row("task-row")])
with TemporaryDirectory() as media_root, self.settings(MEDIA_ROOT=media_root):
file_path = default_storage.save(
"parser_uploads/sanctions.xlsx",
ContentFile(workbook.getvalue()),
)
task_result = parse_ropk_sanctions.apply(
kwargs={
"file_path": file_path,
"original_name": "sanctions.xlsx",
},
task_id="ropk-sanctions-task-test",
)
self.assertFalse(default_storage.exists(file_path))
self.assertTrue(task_result.successful())
self.assertEqual(task_result.result["raw_records_count"], 1)
self.assertEqual(task_result.result["published_records_count"], 1)
self.assertEqual(task_result.result["quarantine_records_count"], 0)
self.assertIn("load_id", task_result.result)
job = BackgroundJob.objects.get(task_id="ropk-sanctions-task-test")
self.assertEqual(job.status, JobStatus.SUCCESS)
self.assertEqual(job.result["published_records_count"], 1)
self.assertEqual(
ParserLoadLog.objects.get(source=ROPK_SANCTIONS_SOURCE).status,
ParserLoadLog.Status.SUCCESS,
)

View File

@@ -86,6 +86,7 @@ class SourceCardServiceUnitTest(SimpleTestCase):
"planned-inspections",
"bankruptcy-procedures",
"defense-unreliable-suppliers",
"ropk-sanctions",
"arbitration-cases",
"information-security-registries",
"labor-vacancies",
@@ -102,6 +103,7 @@ class SourceCardServiceUnitTest(SimpleTestCase):
"Плановые проверки Генпрокуратуры России",
"Сведения о процедурах банкротства",
"Недобросовестные поставщики ГОЗ",
"РОПК — Санкции",
"Арбитражные дела",
"Реестры по информационной безопасности",
"Вакансии Работа России",