feat: add Fedresurs profiles and integrated structures sources
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 6m35s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Failing after 6m35s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
This commit is contained in:
@@ -210,7 +210,7 @@ def test_sro_export_keeps_all_years_and_identifier_strings(export_client, file_f
|
||||
record_date="2001-01-01",
|
||||
)
|
||||
generation = exports.build_source_record_export_artifacts()
|
||||
assert generation.artifacts_count == 43
|
||||
assert generation.artifacts_count == 49
|
||||
response = issue(export_client, "sro_membership", file_format)
|
||||
assert response.status_code == 201
|
||||
package = exports.consume_source_record_export_download_ticket(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Tests for organization source-record ZIP export."""
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
@@ -28,6 +29,7 @@ from organizations.models import (
|
||||
from organizations.source_record_export import (
|
||||
SourceRecordExportArtifactsUnavailable,
|
||||
_render_source_group_artifact,
|
||||
_render_xlsx_files,
|
||||
_source_group_queryset,
|
||||
_spool_source_group_rows,
|
||||
build_source_record_export_artifacts,
|
||||
@@ -71,6 +73,32 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
return b"".join(response.streaming_content)
|
||||
return response.content
|
||||
|
||||
def test_xlsx_export_preserves_large_rendered_fedresurs_field_in_chunks(self):
|
||||
value = "Атрибут 😀" * 6_000
|
||||
with TemporaryDirectory() as directory:
|
||||
spool = Path(directory) / "rows.jsonl"
|
||||
spool.write_text(
|
||||
json.dumps(
|
||||
{"uid": "record-1", "payload.rendered_page.text": value},
|
||||
ensure_ascii=False,
|
||||
) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
paths = _render_xlsx_files(
|
||||
row_spool_path=spool,
|
||||
output_path=Path(directory) / "fedresurs.xlsx",
|
||||
headers=["uid", "payload.rendered_page.text"],
|
||||
records_count=1,
|
||||
)
|
||||
workbook = load_workbook(paths[0], read_only=True)
|
||||
self.assertEqual(workbook["data"]["B2"].value, "См. overflow_values: payload.rendered_page.text")
|
||||
chunks = [
|
||||
row[4][4:]
|
||||
for row in list(workbook["overflow_values_1"].values)[1:]
|
||||
]
|
||||
self.assertEqual(base64.b64decode("".join(chunks)).decode("utf-8"), value)
|
||||
workbook.close()
|
||||
|
||||
def test_export_returns_service_unavailable_before_first_nightly_generation(self):
|
||||
self.client.force_authenticate(UserFactory.create_superuser())
|
||||
|
||||
@@ -93,9 +121,9 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
call_command("build_source_record_exports", stdout=command_output)
|
||||
|
||||
generation = load_current_source_record_export_generation()
|
||||
self.assertEqual(generation.artifacts_count, 43)
|
||||
self.assertEqual(generation.artifacts_count, 49)
|
||||
self.assertEqual(generation.export_year, timezone.localdate().year)
|
||||
self.assertIn('"artifacts_count": 43', command_output.getvalue())
|
||||
self.assertIn('"artifacts_count": 49', command_output.getvalue())
|
||||
|
||||
def test_sanctions_xlsx_export_uses_source_contract_columns_and_boolean_flags(self):
|
||||
organization = Organization.objects.create(
|
||||
@@ -466,7 +494,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
self.assertEqual(
|
||||
response["X-Source-Export-Generated-At"], generation.generated_at
|
||||
)
|
||||
self.assertEqual(generation.artifacts_count, 43)
|
||||
self.assertEqual(generation.artifacts_count, 49)
|
||||
self.assertIn(
|
||||
'filename="planned-inspections__financial-indicators_',
|
||||
response["Content-Disposition"],
|
||||
@@ -661,8 +689,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
key=lambda item: item.file_name,
|
||||
)
|
||||
|
||||
self.assertEqual(generation.artifacts_count, 43)
|
||||
self.assertEqual(generation.files_count, 44)
|
||||
self.assertEqual(generation.artifacts_count, 49)
|
||||
self.assertEqual(generation.files_count, 50)
|
||||
self.assertEqual(
|
||||
[item.file_name for item in artifacts],
|
||||
[
|
||||
@@ -860,7 +888,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
||||
first_generation = build_source_record_export_artifacts()
|
||||
current_generation = load_current_source_record_export_generation()
|
||||
|
||||
self.assertEqual(first_generation.artifacts_count, 43)
|
||||
self.assertEqual(first_generation.artifacts_count, 49)
|
||||
self.assertEqual(
|
||||
current_generation.generation_id, first_generation.generation_id
|
||||
)
|
||||
|
||||
@@ -147,7 +147,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase):
|
||||
result = refresh_source_record_export_artifacts()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["artifacts_count"], 43)
|
||||
self.assertEqual(result["artifacts_count"], 49)
|
||||
self.assertEqual(result["export_year"], timezone.localdate().year)
|
||||
self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))
|
||||
|
||||
|
||||
@@ -66,11 +66,20 @@ class TestCompaniesCommandsTest(TestCase):
|
||||
10,
|
||||
)
|
||||
|
||||
expected_groups = {choice.value for choice in SourceGroup}
|
||||
expected_groups = {
|
||||
choice.value
|
||||
for choice in SourceGroup
|
||||
if choice is not SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS
|
||||
}
|
||||
expected_sources = {
|
||||
choice.value
|
||||
for choice in ParserLoadLog.Source
|
||||
if choice is not ParserLoadLog.Source.ROPK_SANCTIONS
|
||||
if choice
|
||||
not in {
|
||||
ParserLoadLog.Source.ROPK_SANCTIONS,
|
||||
ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS,
|
||||
ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE,
|
||||
}
|
||||
}
|
||||
for company in companies:
|
||||
self.assertEqual(
|
||||
@@ -264,7 +273,7 @@ class TestCompaniesCommandsTest(TestCase):
|
||||
)
|
||||
self.assertEqual(
|
||||
OrganizationSourceRecord.objects.count(),
|
||||
20 * (len(ParserLoadLog.Source) - 1),
|
||||
20 * (len(ParserLoadLog.Source) - 3),
|
||||
)
|
||||
|
||||
def test_create_replaces_stale_source_record_for_same_legacy_row(self):
|
||||
|
||||
146
tests/apps/parsers/test_fedresurs_company_profile.py
Normal file
146
tests/apps/parsers/test_fedresurs_company_profile.py
Normal file
@@ -0,0 +1,146 @@
|
||||
from apps.parsers.fedresurs_browser import _related_company_urls, _wait_public_element
|
||||
from apps.parsers.fedresurs_company_profile import (
|
||||
FedresursUpstreamError,
|
||||
parse_profile_html,
|
||||
parse_related_page_html,
|
||||
parse_search_html,
|
||||
)
|
||||
from django.test import SimpleTestCase
|
||||
|
||||
SEARCH_HTML = """
|
||||
<app-quick-search-entities-result-card>
|
||||
<div>АО Тест</div><div><div>ИНН</div><div>6150061451</div></div>
|
||||
<div><div>ОГРН</div><div>1096100003625</div></div>
|
||||
</app-quick-search-entities-result-card>
|
||||
"""
|
||||
PROFILE_HTML = """
|
||||
<div>АО Тест</div><div>Действующее</div>
|
||||
<div page-item-name="general-info"><div>Общая информация</div>
|
||||
<div><div>ИНН</div><div>6150061451</div></div>
|
||||
<div><div>ОГРН</div><div>1096100003625</div></div>
|
||||
<div><div>КПП</div><div>772601001</div></div>
|
||||
<div><div>Полное наименование</div><div>АКЦИОНЕРНОЕ ОБЩЕСТВО ТЕСТ</div></div>
|
||||
<div><div>Дата регистрации</div><div>13.11.2009</div></div>
|
||||
</div>
|
||||
<div page-item-name="messages"><div>Публикации</div><div>Нет данных</div></div>
|
||||
<div page-item-name="encumbrances"><div>Обременения</div><div>Нет данных</div></div>
|
||||
<div page-item-name="eio"><div>Единоличный исполнительный орган</div><div>Нет данных</div></div>
|
||||
<div page-item-name="sro"><div>Сведения о СРО</div><div>Нет данных</div></div>
|
||||
<div page-item-name="sro-membership"><div>Членство в СРО</div><div>Нет данных</div></div>
|
||||
<div page-item-name="licenses"><div>Лицензии</div><div>Нет данных</div></div>
|
||||
<div page-item-name="networth"><div>Стоимость чистых активов</div><div>Нет данных</div></div>
|
||||
<div page-item-name="reorganization"><div>Сведения о реорганизации</div><div>Нет данных</div></div>
|
||||
<div page-item-name="bankrupt"><div>Сведения о банкротстве</div><div>Нет данных</div></div>
|
||||
<div page-item-name="trades"><div>Торги</div><div>Нет данных</div></div>
|
||||
"""
|
||||
|
||||
|
||||
class FedresursDomTest(SimpleTestCase):
|
||||
def test_exact_search_match(self):
|
||||
result = parse_search_html(SEARCH_HTML, inn="6150061451", ogrn="1096100003625")
|
||||
self.assertEqual(result, "match")
|
||||
|
||||
def test_search_never_accepts_partial_identifier(self):
|
||||
result = parse_search_html(SEARCH_HTML, inn="615006145", ogrn="1096100003625")
|
||||
self.assertEqual(result, "no_match")
|
||||
|
||||
def test_duplicate_exact_search_is_ambiguous(self):
|
||||
result = parse_search_html(SEARCH_HTML + SEARCH_HTML, inn="6150061451", ogrn="1096100003625")
|
||||
self.assertEqual(result, "ambiguous")
|
||||
|
||||
def test_profile_requires_complete_identity_and_sections(self):
|
||||
profile = parse_profile_html(
|
||||
PROFILE_HTML,
|
||||
url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52",
|
||||
inn="6150061451", ogrn="1096100003625",
|
||||
)
|
||||
self.assertEqual(profile["fedresurs_guid"], "e1661e0e-d217-4598-9e70-a94f4d4bbe52")
|
||||
self.assertEqual(profile["profile"]["kpp"], "772601001")
|
||||
self.assertEqual(profile["sections"]["encumbrances"], "empty")
|
||||
self.assertIn("ИНН", profile["rendered_sections"]["profile"]["text"])
|
||||
self.assertIn("tree", profile["rendered_sections"]["profile"])
|
||||
self.assertIn("АКЦИОНЕРНОЕ ОБЩЕСТВО ТЕСТ", profile["rendered_page"]["text"])
|
||||
self.assertIn(
|
||||
{"label": "ИНН", "value": "6150061451"},
|
||||
profile["rendered_sections"]["profile"]["attributes"],
|
||||
)
|
||||
|
||||
def test_profile_keeps_new_unknown_sections_without_schema_changes(self):
|
||||
html = PROFILE_HTML + (
|
||||
'<div page-item-name="additional-attributes">'
|
||||
'<div>Новый атрибут</div><div>Полное значение</div></div>'
|
||||
)
|
||||
profile = parse_profile_html(
|
||||
html,
|
||||
url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52",
|
||||
inn="6150061451",
|
||||
ogrn="1096100003625",
|
||||
)
|
||||
self.assertIn("Полное значение", profile["rendered_page"]["text"])
|
||||
self.assertIn("Полное значение", profile["rendered_sections"]["additional-attributes"]["text"])
|
||||
|
||||
def test_related_message_keeps_all_rendered_fields_and_links(self):
|
||||
page = parse_related_page_html(
|
||||
'<sfact-message-card><div page-item-name="">'
|
||||
'<div>Аудируемое лицо</div><div>АО ЭКОС</div>'
|
||||
'<div><div>Мнение аудитора</div><div>Полный текст заключения</div></div>'
|
||||
'<a href="/documents/a.pdf">Документ</a></div></sfact-message-card>',
|
||||
url="https://fedresurs.ru/sfactmessages/b2beb243-fffb-40c6-8b10-6c890ba59578",
|
||||
page_type="message",
|
||||
)
|
||||
self.assertIn("Полный текст заключения", page["text"])
|
||||
self.assertEqual(page["links"][0]["url"], "https://fedresurs.ru/documents/a.pdf")
|
||||
self.assertIn(
|
||||
{"label": "Мнение аудитора", "value": "Полный текст заключения"},
|
||||
page["attributes"],
|
||||
)
|
||||
self.assertNotIn("_ngcontent", str(page["tree"]))
|
||||
|
||||
def test_only_follows_discovered_same_company_pages(self):
|
||||
profile = {
|
||||
"fedresurs_guid": "e1661e0e-d217-4598-9e70-a94f4d4bbe52",
|
||||
"rendered_sections": {
|
||||
"publications": {"links": [
|
||||
{"url": "https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52/publications"},
|
||||
{"url": "https://fedresurs.ru/companies/00000000-0000-0000-0000-000000000000/publications"},
|
||||
{"url": "https://evil.example/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52/publications"},
|
||||
]},
|
||||
},
|
||||
}
|
||||
self.assertEqual(_related_company_urls(profile), [
|
||||
("publications", "https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52/publications")
|
||||
])
|
||||
|
||||
def test_challenge_and_partial_page_are_upstream_errors(self):
|
||||
with self.assertRaises(FedresursUpstreamError):
|
||||
parse_profile_html("<body>Проверка Qrator CAPTCHA</body>", url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", inn="6150061451", ogrn="1096100003625")
|
||||
with self.assertRaises(FedresursUpstreamError):
|
||||
parse_profile_html(PROFILE_HTML.replace('page-item-name="general-info"', 'data-missing="1"'), url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52", inn="6150061451", ogrn="1096100003625")
|
||||
with self.assertRaisesRegex(FedresursUpstreamError, "upstream_access_blocked"):
|
||||
parse_search_html(
|
||||
"<body><div>403 Error</div><div>Forbidden</div>"
|
||||
"<p>Access to fedresurs.ru is forbidden.</p></body>",
|
||||
inn="6150061451", ogrn="1096100003625",
|
||||
)
|
||||
|
||||
def test_script_mentions_of_challenge_do_not_hide_valid_public_page(self):
|
||||
profile = parse_profile_html(
|
||||
PROFILE_HTML + '<script>const marker = "Qrator CAPTCHA";</script>',
|
||||
url="https://fedresurs.ru/companies/e1661e0e-d217-4598-9e70-a94f4d4bbe52",
|
||||
inn="6150061451",
|
||||
ogrn="1096100003625",
|
||||
)
|
||||
self.assertEqual(profile["profile"]["inn"], "6150061451")
|
||||
|
||||
def test_wait_detects_late_403_instead_of_reporting_timeout(self):
|
||||
class BlockedBrowser:
|
||||
page_source = (
|
||||
"<body><div>403 Error</div>"
|
||||
"<p>Access to fedresurs.ru is forbidden.</p></body>"
|
||||
)
|
||||
|
||||
def find_elements(self, *_args):
|
||||
return []
|
||||
|
||||
with self.assertRaisesRegex(FedresursUpstreamError, "upstream_access_blocked"):
|
||||
_wait_public_element(BlockedBrowser(), "css selector", "input")
|
||||
79
tests/apps/parsers/test_fedresurs_snapshot.py
Normal file
79
tests/apps/parsers/test_fedresurs_snapshot.py
Normal file
@@ -0,0 +1,79 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.parsers.fedresurs_company_profile import FedresursUpstreamError
|
||||
from apps.parsers.fedresurs_snapshot import refresh_fedresurs_company_profiles
|
||||
from django.test import TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
|
||||
|
||||
def profile(guid, inn, ogrn):
|
||||
return {
|
||||
"fedresurs_guid": guid,
|
||||
"source_url": f"https://fedresurs.ru/companies/{guid}",
|
||||
"profile": {
|
||||
"full_name": "АО Тест",
|
||||
"registration_date": "2020-01-01",
|
||||
"status": {"is_active": True},
|
||||
"inn": inn,
|
||||
"ogrn": ogrn,
|
||||
},
|
||||
"sections": {"bankruptcy": "empty"},
|
||||
}
|
||||
|
||||
|
||||
@override_settings(
|
||||
FEDRESURS_COMPANY_PROFILE_ENABLED=True,
|
||||
FEDRESURS_MIN_INTERVAL_SECONDS=0,
|
||||
FEDRESURS_MAX_RETRIES=0,
|
||||
)
|
||||
class FedresursSnapshotTest(TestCase):
|
||||
def setUp(self):
|
||||
self.first = Organization.objects.create(
|
||||
name="АО Первый", inn="6150061451", ogrn="1096100003625",
|
||||
okpo="00084008", directory_imported_at=timezone.now(),
|
||||
opk_registry_membership=True,
|
||||
)
|
||||
self.second = Organization.objects.create(
|
||||
name="АО Второй", inn="1234567890", ogrn="1027700132195",
|
||||
okpo="00084009", directory_imported_at=timezone.now(),
|
||||
opk_registry_membership=True,
|
||||
)
|
||||
|
||||
@patch("apps.parsers.fedresurs_snapshot.public_browser")
|
||||
@patch("apps.parsers.fedresurs_snapshot.lookup_company")
|
||||
def test_complete_batch_publishes_and_targeted_refresh_preserves_other_records(self, lookup, browser):
|
||||
browser.return_value.__enter__.return_value = object()
|
||||
lookup.side_effect = lambda driver, *, inn, ogrn: (
|
||||
"match", profile(
|
||||
"e1661e0e-d217-4598-9e70-a94f4d4bbe52" if inn == self.first.inn
|
||||
else "a1661e0e-d217-4598-9e70-a94f4d4bbe52", inn, ogrn,
|
||||
)
|
||||
)
|
||||
_, result = refresh_fedresurs_company_profiles(load_batch=1)
|
||||
self.assertEqual(result.published, 2)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 2)
|
||||
lookup.return_value = ("no_match", None)
|
||||
lookup.side_effect = None
|
||||
_, targeted = refresh_fedresurs_company_profiles(
|
||||
load_batch=2, mode="targeted", organization_uids=[str(self.first.uid)]
|
||||
)
|
||||
self.assertEqual(targeted.quarantined, 1)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 1)
|
||||
|
||||
@patch("apps.parsers.fedresurs_snapshot.public_browser")
|
||||
@patch("apps.parsers.fedresurs_snapshot.lookup_company")
|
||||
def test_technical_failure_keeps_last_published_snapshot(self, lookup, browser):
|
||||
browser.return_value.__enter__.return_value = object()
|
||||
lookup.side_effect = lambda driver, *, inn, ogrn: (
|
||||
"match", profile(
|
||||
"e1661e0e-d217-4598-9e70-a94f4d4bbe52" if inn == self.first.inn
|
||||
else "a1661e0e-d217-4598-9e70-a94f4d4bbe52", inn, ogrn,
|
||||
)
|
||||
)
|
||||
refresh_fedresurs_company_profiles(load_batch=1)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 2)
|
||||
lookup.side_effect = FedresursUpstreamError("upstream_challenge")
|
||||
with self.assertRaises(FedresursUpstreamError):
|
||||
refresh_fedresurs_company_profiles(load_batch=2)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="fedresurs_company_profile").count(), 2)
|
||||
282
tests/apps/parsers/test_integrated_structure_workbook.py
Normal file
282
tests/apps/parsers/test_integrated_structure_workbook.py
Normal file
@@ -0,0 +1,282 @@
|
||||
from io import BytesIO
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.parsers.integrated_structure_workbook import (
|
||||
IntegratedWorkbookError,
|
||||
parse_integrated_workbook,
|
||||
)
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.test import SimpleTestCase, TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
from openpyxl import Workbook
|
||||
from organizations.models import Organization, OrganizationSourceRecord
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from tests.apps.user.factories import UserFactory
|
||||
|
||||
|
||||
def workbook_bytes(*, okpo="00084008", doc_id=93, document_stage=1137):
|
||||
workbook = Workbook()
|
||||
stages = workbook.active
|
||||
stages.title = "is_stages"
|
||||
stages.append(["okpo", "stage_id", "stage_date", "stage_prim"])
|
||||
stages.append([okpo, 1137, "2026-09-01", "Первое"])
|
||||
stages.append([okpo, 1245, "2026-09-02", None])
|
||||
documents = workbook.create_sheet("is_stages_docs")
|
||||
documents.append(
|
||||
[
|
||||
"okpo", "stage_id", "stage_date", "doc_id", "doc_name",
|
||||
"doc_type", "doc_n", "doc_date", "dsp", "have_text",
|
||||
"cons_local", "cons_web",
|
||||
]
|
||||
)
|
||||
documents.append(
|
||||
[
|
||||
okpo, document_stage, "2026-09-01", doc_id, "Документ",
|
||||
"Приказ", "001", "2026-09-01", 0, 1, "local://not-for-browser",
|
||||
"https://online.consultant.ru/example",
|
||||
]
|
||||
)
|
||||
stream = BytesIO()
|
||||
workbook.save(stream)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
class IntegratedWorkbookTest(SimpleTestCase):
|
||||
def test_parses_relations_and_preserves_zero_padded_okpo(self):
|
||||
result = parse_integrated_workbook(BytesIO(workbook_bytes()))
|
||||
self.assertEqual(result.counts, (1, 2, 1, 1, 1))
|
||||
self.assertEqual(result.structures["00084008"].events[0].external_id, "1137")
|
||||
self.assertEqual(result.structures["00084008"].events[1].documents, [])
|
||||
self.assertEqual(result.documents["93"].number, "001")
|
||||
self.assertEqual(
|
||||
result.documents["93"].consultant_web_url,
|
||||
"https://online.consultant.ru/example",
|
||||
)
|
||||
|
||||
def test_rejects_numeric_okpo_because_leading_zeroes_are_unrecoverable(self):
|
||||
with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_okpo"):
|
||||
parse_integrated_workbook(BytesIO(workbook_bytes(okpo=84008)))
|
||||
|
||||
def test_rejects_orphan_document_relation(self):
|
||||
with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"):
|
||||
parse_integrated_workbook(
|
||||
BytesIO(workbook_bytes(document_stage=9999))
|
||||
)
|
||||
|
||||
def test_rejects_non_allowlisted_document_url(self):
|
||||
value = workbook_bytes()
|
||||
workbook = Workbook()
|
||||
# Use an ordinary generated fixture rather than modifying ZIP internals.
|
||||
from openpyxl import load_workbook
|
||||
|
||||
workbook = load_workbook(BytesIO(value))
|
||||
workbook["is_stages_docs"]["L2"] = "https://attacker.example/file"
|
||||
stream = BytesIO()
|
||||
workbook.save(stream)
|
||||
with self.assertRaisesRegex(IntegratedWorkbookError, "invalid_consultant_url"):
|
||||
parse_integrated_workbook(BytesIO(stream.getvalue()))
|
||||
|
||||
def test_rejects_duplicate_stage(self):
|
||||
from openpyxl import load_workbook
|
||||
|
||||
workbook = load_workbook(BytesIO(workbook_bytes()))
|
||||
workbook["is_stages"].append(["00084008", 1137, "2026-09-01", "duplicate"])
|
||||
stream = BytesIO()
|
||||
workbook.save(stream)
|
||||
with self.assertRaisesRegex(IntegratedWorkbookError, "duplicate_stage"):
|
||||
parse_integrated_workbook(BytesIO(stream.getvalue()))
|
||||
|
||||
|
||||
class IntegratedStructureImportTest(TestCase):
|
||||
def setUp(self):
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
self.private = TemporaryDirectory()
|
||||
self.addCleanup(self.private.cleanup)
|
||||
self.settings = override_settings(
|
||||
PARSER_PRIVATE_ARTIFACT_ROOT=self.private.name,
|
||||
MEDIA_ROOT=self.private.name + "/public",
|
||||
)
|
||||
self.settings.enable()
|
||||
self.addCleanup(self.settings.disable)
|
||||
Organization.objects.create(
|
||||
name="АО Тест", inn="1234567890", ogrn="1027700132195",
|
||||
okpo="00084008", directory_imported_at=timezone.now(),
|
||||
)
|
||||
|
||||
def test_atomic_publish_relations_attachment_and_idempotency(self):
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
from apps.parsers.models import (
|
||||
IntegratedStructureAttachment,
|
||||
IntegratedStructureDocument,
|
||||
IntegratedStructureEvent,
|
||||
IntegratedStructureEventDocument,
|
||||
)
|
||||
|
||||
workbook = workbook_bytes()
|
||||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||||
attachment.name = "000093.rtf"
|
||||
first = import_integrated_structures(
|
||||
workbook=BytesIO(workbook), original_name="test.xlsx",
|
||||
attachments=[attachment], uploaded_by_id=None,
|
||||
)
|
||||
second = import_integrated_structures(
|
||||
workbook=BytesIO(workbook), original_name="test.xlsx",
|
||||
attachments=[attachment], uploaded_by_id=None,
|
||||
)
|
||||
self.assertEqual(first[0].uid, second[0].uid)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1)
|
||||
self.assertEqual(IntegratedStructureEvent.objects.count(), 2)
|
||||
self.assertEqual(IntegratedStructureDocument.objects.count(), 1)
|
||||
self.assertEqual(IntegratedStructureEventDocument.objects.count(), 1)
|
||||
self.assertEqual(IntegratedStructureAttachment.objects.count(), 1)
|
||||
|
||||
def test_text_document_id_keeps_leading_zeroes_for_attachment(self):
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
from apps.parsers.models import IntegratedStructureDocument
|
||||
|
||||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||||
attachment.name = "000093.rtf"
|
||||
import_integrated_structures(
|
||||
workbook=BytesIO(workbook_bytes(doc_id="000093")),
|
||||
original_name="test.xlsx",
|
||||
attachments=[attachment],
|
||||
)
|
||||
document = IntegratedStructureDocument.objects.get(external_id="000093")
|
||||
self.assertEqual(document.attachment_filename, "000093.rtf")
|
||||
self.assertIsNotNone(document.attachment)
|
||||
|
||||
def test_invalid_batch_retains_last_snapshot(self):
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
|
||||
valid = workbook_bytes()
|
||||
import_integrated_structures(
|
||||
workbook=BytesIO(valid), original_name="test.xlsx", attachments=[]
|
||||
)
|
||||
with self.assertRaisesRegex(IntegratedWorkbookError, "unknown_stage"):
|
||||
import_integrated_structures(
|
||||
workbook=BytesIO(workbook_bytes(document_stage=9999)),
|
||||
original_name="bad.xlsx", attachments=[],
|
||||
)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 1)
|
||||
|
||||
def test_failed_publication_preserves_rejected_artifact_and_failed_log(self):
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact
|
||||
|
||||
with patch(
|
||||
"apps.parsers.integrated_structures._publish_relations",
|
||||
side_effect=RuntimeError("storage_failure"),
|
||||
), self.assertRaisesRegex(RuntimeError, "storage_failure"):
|
||||
import_integrated_structures(
|
||||
workbook=BytesIO(workbook_bytes()),
|
||||
original_name="test.xlsx",
|
||||
attachments=[],
|
||||
)
|
||||
artifact = ParserSourceArtifact.objects.get(source="integrated_structure_documents")
|
||||
load_log = ParserLoadLog.objects.get(source="integrated_structure_documents")
|
||||
self.assertEqual(artifact.status, ParserSourceArtifact.Status.REJECTED)
|
||||
self.assertEqual(load_log.status, ParserLoadLog.Status.FAILED)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.filter(source="integrated_structures").count(), 0)
|
||||
|
||||
def test_export_has_three_relational_sheets_without_binary(self):
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
from openpyxl import load_workbook
|
||||
from organizations.source_record_export import (
|
||||
build_source_record_export_artifacts,
|
||||
)
|
||||
|
||||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||||
attachment.name = "000093.rtf"
|
||||
import_integrated_structures(
|
||||
workbook=BytesIO(workbook_bytes()), original_name="test.xlsx",
|
||||
attachments=[attachment],
|
||||
)
|
||||
with TemporaryDirectory() as directory:
|
||||
generation = build_source_record_export_artifacts(export_directory=directory)
|
||||
artifact = next(
|
||||
item for item in generation.artifacts
|
||||
if item.source_group == "integrated_structure_documents" and item.file_format == "xlsx"
|
||||
)
|
||||
workbook = load_workbook(artifact.path, read_only=True)
|
||||
self.assertEqual(workbook.sheetnames, ["structures", "events", "event_documents"])
|
||||
self.assertEqual(sum(1 for _ in workbook["structures"].iter_rows()), 2)
|
||||
self.assertEqual(sum(1 for _ in workbook["events"].iter_rows()), 3)
|
||||
self.assertEqual(sum(1 for _ in workbook["event_documents"].iter_rows()), 2)
|
||||
self.assertNotIn(b"{\\rtf1", artifact.path.read_bytes())
|
||||
workbook.close()
|
||||
|
||||
def test_detail_and_authorized_attachment_stream(self):
|
||||
from apps.parsers.integrated_structures import import_integrated_structures
|
||||
|
||||
attachment = BytesIO(b"{\\rtf1\\ansi document}")
|
||||
attachment.name = "000093.rtf"
|
||||
import_integrated_structures(
|
||||
workbook=BytesIO(workbook_bytes()), original_name="test.xlsx",
|
||||
attachments=[attachment],
|
||||
)
|
||||
record = OrganizationSourceRecord.objects.get(source="integrated_structures")
|
||||
path = f"/api/v2/organization-source-records/{record.uid}/"
|
||||
download = path + "documents/93/attachment/"
|
||||
client = APIClient()
|
||||
self.assertEqual(client.get(download).status_code, 401)
|
||||
with override_settings(ORGANIZATIONS_V2_ALLOW_ANONYMOUS=True):
|
||||
self.assertEqual(client.get(download).status_code, 401)
|
||||
client.force_authenticate(UserFactory.create_user())
|
||||
detail = client.get(path)
|
||||
self.assertEqual(detail.status_code, 200)
|
||||
payload = detail.data["payload"]
|
||||
self.assertEqual(payload["structure"]["events_count"], 2)
|
||||
self.assertEqual(len(payload["events"]), 2)
|
||||
self.assertEqual(payload["events"][1]["documents"][0]["attachment"]["filename"], "000093.rtf")
|
||||
response = client.get(download)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
self.assertEqual(b"".join(response.streaming_content), b"{\\rtf1\\ansi document}")
|
||||
|
||||
def test_upload_requires_admin_and_reports_missing_rtf(self):
|
||||
user = UserFactory.create_user()
|
||||
admin = UserFactory.create_superuser()
|
||||
client = APIClient()
|
||||
url = "/api/v1/parsers/upload/integrated_structure_documents/"
|
||||
client.force_authenticate(user)
|
||||
response = client.post(
|
||||
url,
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(response.status_code, 403)
|
||||
client.force_authenticate(admin)
|
||||
response = client.post(
|
||||
url,
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(response.status_code, 200)
|
||||
record = OrganizationSourceRecord.objects.get(source="integrated_structures")
|
||||
client.force_authenticate(user)
|
||||
missing = client.get(
|
||||
f"/api/v2/organization-source-records/{record.uid}/documents/93/attachment/"
|
||||
)
|
||||
self.assertEqual(missing.status_code, 409)
|
||||
self.assertEqual(missing.data["errors"][0]["code"], "attachment_unavailable")
|
||||
|
||||
def test_unexpected_upload_failure_finishes_background_job(self):
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
|
||||
client = APIClient()
|
||||
client.raise_request_exception = False
|
||||
client.force_authenticate(UserFactory.create_superuser())
|
||||
with patch(
|
||||
"apps.parsers.integrated_structures.import_integrated_structures",
|
||||
side_effect=RuntimeError("storage_failure"),
|
||||
):
|
||||
response = client.post(
|
||||
"/api/v1/parsers/upload/integrated_structure_documents/",
|
||||
{"workbook": SimpleUploadedFile("test.xlsx", workbook_bytes())},
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(response.status_code, 500)
|
||||
self.assertEqual(BackgroundJob.objects.get().status, JobStatus.FAILURE)
|
||||
@@ -95,6 +95,8 @@ class SourceCardServiceUnitTest(TestCase):
|
||||
"sme-support-recipients-registry",
|
||||
"budget-process-registry",
|
||||
"sro-membership-check",
|
||||
"integrated-structure-documents",
|
||||
"fedresurs-company-profile",
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
@@ -115,6 +117,8 @@ class SourceCardServiceUnitTest(TestCase):
|
||||
"Реестр субъектов МСП — получателей поддержки",
|
||||
"Реестр участников бюджетного процесса",
|
||||
"Проверка членства в СРО",
|
||||
"Документы интегрированных структур",
|
||||
"Федресурс: сведения о юридических лицах",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user