fix: implement backend testing amendments and atomic source publication
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m16s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled

This commit is contained in:
Aleksandr Meshchryakov
2026-10-12 09:47:28 +02:00
parent 2eeb237a47
commit 9b3da5b505
46 changed files with 2918 additions and 335 deletions

View File

@@ -193,3 +193,22 @@ def test_fssp_variant_preserves_signed_nullable_amounts(contract):
"ended",
"terminated",
]
def test_testing_amendments_download_and_options_contract(contract):
download = contract["paths"][
"/api/v2/organization-source-records/export-download/"
]["post"]
assert set(download["requestBody"]["content"]) == {
"application/x-www-form-urlencoded"
}
assert set(download["responses"]["200"]["content"]) == {"application/zip"}
invalid = download["responses"]["400"]["content"]["application/json"]["schema"]
schema = contract["components"]["schemas"][invalid["$ref"].rsplit("/", 1)[1]]
assert schema["properties"]["code"]["enum"] == ["invalid_ticket"]
options = contract["paths"]["/api/v2/organization-source-records/filter-options/"][
"get"
]
assert {
param["name"] for param in options["parameters"] if param.get("required")
} == {"source_group", "field"}

View File

@@ -178,7 +178,7 @@ class CheckoSectionContractTests(TestCase):
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_task(source)
job = BackgroundJob.objects.get()
assert job.status == "failure"
assert job.status == "failed"
assert job.result["quota_skipped"] == 1
assert job.result["organizations_preserved"] == 1
assert job.result["completeness"] == "partial"

View File

@@ -9,6 +9,7 @@ from unittest.mock import patch
from uuid import uuid4
import pytest
from apps.core.models import BackgroundJob
from apps.parsers.budget_registry import normalize_budget_record
from apps.parsers.checko_sections import collect_checko_section
from apps.parsers.fssp_csv import FSSP_SOURCE, GENERAL_DATASET, iter_fssp_records
@@ -333,6 +334,31 @@ def test_source_record_runtime_fixtures():
404,
)
options_url = list_url + "filter-options/"
capture(
"message-type-options",
"options",
client.get(options_url, {"source_group": "leasing", "field": "message_type"}),
200,
)
capture(
"message-type-options-invalid",
"options",
client.get(options_url, {"source_group": "unknown", "field": "message_type"}),
400,
)
user = UserFactory.create_superuser()
client.force_authenticate(user)
for state in ("failed", "success", "skipped"):
job = BackgroundJob.objects.create(
task_id=f"contract-{state}", task_name="test", user_id=user.id
)
if state == "failed":
job.fail("HTTPError: https://user:secret@example.test/?token=secret")
else:
job.complete(result={"status": state})
capture(f"job-{state}", "job", client.get(f"/api/v1/jobs/{job.task_id}/"), 200)
# Ordinary pytest runs create no artifact. CI opts in with an explicit path.
destination = os.environ.get("OPENAPI_RUNTIME_FIXTURES")
if destination:

View File

@@ -158,7 +158,7 @@ def test_ticket_expiry_and_reuse_are_typed_410(export_client):
assert repeated.status_code == 410
assert repeated.data["code"] == "ticket_expired"
pending = exports.create_source_record_export_download_ticket(
source_groups=["budget_process_registry"], export_format="json"
source_groups=["budget_process_registry"], export_format="json", user_id=1
)
with patch.object(exports.time, "time", return_value=time.time() + 301):
expired = export_client.post(download_url(), {"ticket": pending.ticket})

View File

@@ -723,11 +723,12 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
with self.assertNumQueries(0):
download_response = self.client.post(
self.download_url,
{"ticket": ticket_response.data["ticket"]},
format="multipart",
"ticket=" + ticket_response.data["ticket"],
content_type="application/x-www-form-urlencoded",
)
self.assertEqual(download_response.status_code, status.HTTP_200_OK)
self.assertEqual(download_response["Cache-Control"], "no-store")
self.assertTrue(download_response.streaming)
self.assertEqual(download_response["Content-Type"], "application/zip")
self.assertNotIn("Content-Length", download_response)
@@ -751,6 +752,72 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
"ticket_expired",
)
def test_all_sources_filename_is_short_and_invalid_ticket_is_400(self):
from organizations.source_record_export import (
_build_source_records_archive_name,
)
name = _build_source_records_archive_name(
source_groups=SourceGroup.values,
requested_at=datetime(2026, 10, 9, tzinfo=UTC),
)
self.assertRegex(name, r"^organization_source_records_all_.*\.zip$")
self.assertLess(len(name), 100)
self.client.force_authenticate(user=None)
response = self.client.post(
self.download_url,
"ticket=" + "x" * 43,
content_type="application/x-www-form-urlencoded",
)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.data["code"], "invalid_ticket")
def test_download_not_ready_does_not_consume_ticket(self):
from organizations.source_record_export import (
SourceRecordExportArtifactsUnavailable,
)
self.client.force_authenticate(UserFactory.create_superuser())
build_source_record_export_artifacts()
response = self.client.post(
self.ticket_url,
{"sources": ["planned_inspections"], "format": "json"},
format="json",
)
self.assertEqual(response.status_code, 201)
ticket = response.data["ticket"]
self.client.force_authenticate(user=None)
with patch(
"organizations.source_record_export.build_source_records_export_archive",
side_effect=SourceRecordExportArtifactsUnavailable,
):
unavailable = self.client.post(self.download_url, {"ticket": ticket})
self.assertEqual(unavailable.status_code, 503)
ready = self.client.post(self.download_url, {"ticket": ticket})
self.assertEqual(ready.status_code, 200)
self._response_body(ready)
def test_all_sources_native_form_download_can_be_repeated_with_new_ticket(self):
build_source_record_export_artifacts()
for _ in range(2):
self.client.force_authenticate(UserFactory.create_superuser())
response = self.client.post(
self.ticket_url,
{"sources": SourceGroup.values, "format": "json"},
format="json",
)
self.assertEqual(response.status_code, 201)
self.assertLess(len(response.data["file_name"]), 100)
self.client.force_authenticate(user=None)
download = self.client.post(
self.download_url,
"ticket=" + response.data["ticket"],
content_type="application/x-www-form-urlencoded",
)
self.assertEqual(download.status_code, 200)
with zipfile.ZipFile(BytesIO(self._response_body(download))) as archive:
self.assertEqual(len(archive.namelist()), len(SourceGroup.values))
def test_csv_export_uses_bom_and_canonical_columns_before_payload(self):
self.client.force_authenticate(UserFactory.create_superuser())
organization = Organization.objects.create(

View File

@@ -0,0 +1,387 @@
"""Regression cases from the 8 October backend acceptance report."""
import json
from io import StringIO
from uuid import UUID
import pytest
from django.core.management import call_command
from django.utils import timezone
from organizations.models import (
Organization,
OrganizationSourceExtension,
OrganizationSourceRecord,
)
from organizations.source_ingestion import (
OrganizationSourceIngestionService,
SourceRecordInput,
)
from organizations.source_record_export import _build_record_row
from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
LIST_URL = "/api/v2/organization-source-records/"
@pytest.fixture
def client(db):
client = APIClient()
client.force_authenticate(UserFactory.create_user())
return client
def record(group, source, index, payload=None, **organization_fields):
supplied = payload or {}
if source == "checko_licenses":
payload = {
"license_number": "123",
"issued_on": "2026-01-01",
"valid_from": None,
"valid_until": None,
"authority": "Authority",
"activities": [],
"source_url": "https://example.test",
"fetched_at": "2026-10-09T00:00:00Z",
"upstream": {},
**supplied,
}
if payload["valid_from"] == "":
payload["valid_from"] = None
elif source in ("checko_leasing", "checko_messages"):
payload = {
"message_guid": str(index),
"publication_raw": None,
"publication_date": None,
"message_type": "",
"message_type_name": None,
"publisher_identifier": None,
"participant_identifiers": [],
"source_url": None,
"fetched_at": "2026-10-09T00:00:00Z",
"upstream": {},
"lease_event_type": "test",
**supplied,
}
elif source == "stock_exchange_activity":
payload = {
"exchange": "moex",
"instrument_id": str(index),
"issuer_inn": "7707083810",
"issuer_provider_id": None,
"isin": None,
"instrument_kind": "stock",
"listing_date": None,
"nominal_value": None,
"nominal_currency": None,
"parameters": {},
"duration": None,
"yield": None,
"provenance": {},
"upstream": {},
**supplied,
}
payload["trading"] = {"board_id": "test", "upstream": {}, **payload["trading"]}
organization = Organization.objects.create(
name=f"Organization {index}",
opk_registry_membership=True,
**organization_fields,
)
extension = OrganizationSourceExtension.objects.create(
organization=organization, source_group=group
)
return OrganizationSourceRecord.objects.create(
uid=UUID(int=index),
extension=extension,
source=source,
record_type="test",
external_id=str(index),
payload=payload or {},
record_date="2026-01-01",
)
@pytest.mark.parametrize(
"ordering,expected",
[
("organization__is_branch", [3, 1, 2]),
("-organization__is_branch", [1, 3, 2]),
],
)
@pytest.mark.parametrize(
"group,source",
[
("government_procurements", "procurements_44fz"),
("industrial_production", "industrial"),
("planned_inspections", "inspections"),
("defense_suppliers", "fas_goz"),
("arbitration", "arbitration"),
("security_registries", "fstec"),
("vacancies", "trudvsem"),
],
)
def test_branch_ordering_is_boolean_null_last_before_pagination(
client, ordering, expected, group, source
):
record(group, source, 1, is_branch=True)
record(group, source, 2, is_branch=None)
record(group, source, 3, is_branch=False)
for page, expected_id in enumerate(expected, 1):
response = client.get(
LIST_URL, {"ordering": ordering, "page_size": 1, "page": page}
)
assert response.status_code == 200, response.data
assert response.data["data"][0]["external_id"] == str(expected_id)
@pytest.mark.parametrize(
"group,source,prefix,payloads",
[
(
"integrated_structure_documents",
"integrated_structures",
"first_event_date",
[
{"structure": {"first_event_date": date}}
for date in ["2026-10-08", "2026-10-09", None]
],
),
(
"company_licenses",
"checko_licenses",
"license_valid_from",
[{"valid_from": date} for date in ["2026-10-08", "2026-10-09", None]],
),
(
"stock_exchange_activity",
"stock_exchange_activity",
"traded_on",
[
{"exchange": exchange, "trading": {"traded_on": date}}
for exchange, date in [
("moex", "2026-10-08"),
("spb", "2026-10-09"),
("moex", None),
]
],
),
],
)
def test_date_ranges_use_business_date_and_include_boundaries(
client, group, source, prefix, payloads
):
for index, payload in enumerate(payloads, 1):
record(group, source, index, payload)
response = client.get(
LIST_URL,
{
"source_group": group,
f"{prefix}_from": "2026-10-08",
f"{prefix}_to": "2026-10-09",
"ordering": "uid",
},
)
assert response.status_code == 200, response.data
assert [item["external_id"] for item in response.data["data"]] == ["1", "2"]
for params, code in [
(
{f"{prefix}_from": "2026-10-10", f"{prefix}_to": "2026-10-09"},
"invalid_date_range",
),
({f"{prefix}_from": "09.10.2026"}, "invalid_date"),
(
{"source_group": "arbitration", f"{prefix}_from": "2026-10-09"},
"invalid_filter",
),
]:
invalid = client.get(LIST_URL, {"source_group": group, **params})
assert invalid.status_code == 400
assert code in [error["code"] for error in invalid.data["errors"]]
def test_message_type_options_count_the_same_visible_records(client):
record(
"leasing",
"checko_leasing",
1,
{"message_type": "Lease", "message_type_name": "Лизинг"},
)
record(
"leasing",
"checko_leasing",
2,
{"message_type": "Lease", "message_type_name": "Лизинг"},
)
record("leasing", "checko_leasing", 3, {"message_type": "Other"})
record("company_messages", "checko_messages", 4, {"message_type": "Lease"})
response = client.get(
LIST_URL + "filter-options/",
{"source_group": "leasing", "field": "message_type", "search": "Лиз"},
)
assert response.status_code == 200, response.data
assert response.data["data"] == [
{"value": "Lease", "label": "Лизинг", "records_count": 2}
]
filtered = client.get(
LIST_URL, {"source_group": "leasing", "message_type": "Lease"}
)
assert filtered.data["meta"]["pagination"]["total_count"] == 2
assert (
client.get(
LIST_URL + "filter-options/",
{"source_group": "arbitration", "field": "message_type"},
).status_code
== 400
)
@pytest.mark.parametrize(
"source,group,key",
[
("industrial", "industrial_production", "issue_date_normalized"),
("industrial", "industrial_production", "expiry_date_normalized"),
("fstec", "security_registries", "term_date"),
("checko_licenses", "company_licenses", "valid_from"),
],
)
def test_normalized_date_ordering_null_last(client, source, group, key):
for index, value in enumerate(["2027-01-01", None, "2026-12-31", ""], 1):
record(group, source, index, {key: value})
invalid = client.get(
LIST_URL, {"source_group": "arbitration", "ordering": "payload__" + key}
)
assert invalid.status_code == 400
for ordering, expected in [(key, [3, 1, 2, 4]), ("-" + key, [1, 3, 2, 4])]:
response = client.get(
LIST_URL,
{
"source_group": group,
"source": source,
"ordering": ("-" if ordering.startswith("-") else "")
+ "payload__"
+ key,
},
)
assert response.status_code == 200, response.data
assert [int(item["external_id"]) for item in response.data["data"]] == expected
@pytest.mark.parametrize(
"raw_key", ["Срок действия сертификата", "Срок действия лицензии", "Срок действия"]
)
def test_fstec_ingestion_publishes_normalized_term_without_losing_raw(client, raw_key):
organization = Organization.objects.create(
name="FSTEC",
inn="7707083810",
directory_imported_at=timezone.now(),
opk_registry_membership=True,
)
OrganizationSourceIngestionService.save_records(
source="fstec",
load_batch=1,
records=[
SourceRecordInput(
external_id="term",
title="Term",
organization_name="FSTEC",
inn=organization.inn,
payload={raw_key: "31.12.2027"},
)
],
)
saved = OrganizationSourceRecord.objects.get()
assert saved.payload["term_date"] == "2027-12-31"
assert saved.payload["term_raw"] == "31.12.2027"
assert saved.payload[raw_key] == "31.12.2027"
for path in [LIST_URL, LIST_URL + str(saved.uid) + "/"]:
response = client.get(path)
assert response.status_code == 200
data = response.data["data"] if path == LIST_URL else response.data
item = data[0] if isinstance(data, list) else data
assert item["payload"]["term_date"] == "2027-12-31"
assert (
_build_record_row(saved, include_financial_lines=False)["payload.term_date"]
== "2027-12-31"
)
def test_normalization_backfill_is_dry_run_idempotent_and_preserves_identity(
client, caplog
):
saved = record(
"security_registries",
"fstec",
1,
{"Срок действия": "неизвестен", "filial": ".F."},
filial=".T.",
is_branch=None,
)
original_updated_at = saved.updated_at
output = StringIO()
call_command("normalize_testing_amendments", stdout=output)
assert json.loads(output.getvalue())["changed_organizations"] == 1
saved.refresh_from_db()
assert "term_date" not in saved.payload
call_command("normalize_testing_amendments", apply=True, stdout=StringIO())
saved.refresh_from_db()
saved.extension.organization.refresh_from_db()
assert saved.extension.organization.is_branch is True
assert saved.payload["term_date"] is None
assert saved.payload["term_raw"] == "неизвестен"
assert (
_build_record_row(saved, include_financial_lines=False)[
"organization.is_branch"
]
is True
)
assert saved.payload["is_branch"] is True
assert saved.payload["filial"] == ".F."
assert saved.updated_at == original_updated_at
assert "invalid_date" in caplog.text
assert "branch_conflict" in caplog.text
second = StringIO()
call_command("normalize_testing_amendments", apply=True, stdout=second)
assert json.loads(second.getvalue())["changed_records"] == 0
def test_industrial_ingestion_normalizes_dates_and_logs_invalid_values(client, caplog):
organization = Organization.objects.create(
name="Industrial", inn="7707083810", directory_imported_at=timezone.now()
)
OrganizationSourceIngestionService.save_records(
source="industrial",
load_batch=1,
records=[
SourceRecordInput(
external_id="certificate",
title="Certificate",
organization_name="Industrial",
inn=organization.inn,
payload={"issue_date": "31.12.2025", "expiry_date": "wrong"},
)
],
)
saved = OrganizationSourceRecord.objects.get()
assert saved.payload["issue_date_normalized"] == "2025-12-31"
assert saved.payload["expiry_date_normalized"] is None
assert saved.payload["expiry_date"] == "wrong"
assert "invalid_date" in caplog.text
@pytest.mark.parametrize(
"group,source",
[("government_procurements", "procurements_44fz"), ("arbitration", "arbitration")],
)
@pytest.mark.parametrize(
"ordering,expected", [("amount", [2, 1, 3]), ("-amount", [1, 2, 3])]
)
def test_amount_ordering_remains_decimal_null_last(
client, group, source, ordering, expected
):
for index, amount in enumerate(["100.00", "20.00", None], 1):
saved = record(group, source, index)
saved.amount = amount
saved.save(update_fields=["amount"])
response = client.get(LIST_URL, {"source_group": group, "ordering": ordering})
assert response.status_code == 200, response.data
assert [int(item["external_id"]) for item in response.data["data"]] == expected