"""Regression cases from the 8 October backend acceptance report.""" import json from io import StringIO from uuid import UUID import pytest from django.core.management import call_command from django.utils import timezone from organizations.models import ( Organization, OrganizationSourceExtension, OrganizationSourceRecord, ) from organizations.source_ingestion import ( OrganizationSourceIngestionService, SourceRecordInput, ) from organizations.source_record_export import _build_record_row from rest_framework.test import APIClient from tests.apps.user.factories import UserFactory LIST_URL = "/api/v2/organization-source-records/" @pytest.fixture def client(db): client = APIClient() client.force_authenticate(UserFactory.create_user()) return client def record(group, source, index, payload=None, **organization_fields): supplied = payload or {} if source == "checko_licenses": payload = { "license_number": "123", "issued_on": "2026-01-01", "valid_from": None, "valid_until": None, "authority": "Authority", "activities": [], "source_url": "https://example.test", "fetched_at": "2026-10-09T00:00:00Z", "upstream": {}, **supplied, } if payload["valid_from"] == "": payload["valid_from"] = None elif source in ("checko_leasing", "checko_messages"): payload = { "message_guid": str(index), "publication_raw": None, "publication_date": None, "message_type": "", "message_type_name": None, "publisher_identifier": None, "participant_identifiers": [], "source_url": None, "fetched_at": "2026-10-09T00:00:00Z", "upstream": {}, "lease_event_type": "test", **supplied, } elif source == "stock_exchange_activity": payload = { "exchange": "moex", "instrument_id": str(index), "issuer_inn": "7707083810", "issuer_provider_id": None, "isin": None, "instrument_kind": "stock", "listing_date": None, "nominal_value": None, "nominal_currency": None, "parameters": {}, "duration": None, "yield": None, "provenance": {}, "upstream": {}, **supplied, } payload["trading"] = {"board_id": "test", "upstream": {}, **payload["trading"]} organization = Organization.objects.create( name=f"Organization {index}", opk_registry_membership=True, **organization_fields, ) extension = OrganizationSourceExtension.objects.create( organization=organization, source_group=group ) return OrganizationSourceRecord.objects.create( uid=UUID(int=index), extension=extension, source=source, record_type="test", external_id=str(index), payload=payload or {}, record_date="2026-01-01", ) @pytest.mark.parametrize( "ordering,expected", [ ("organization__is_branch", [3, 1, 2]), ("-organization__is_branch", [1, 3, 2]), ], ) @pytest.mark.parametrize( "group,source", [ ("government_procurements", "procurements_44fz"), ("industrial_production", "industrial"), ("planned_inspections", "inspections"), ("defense_suppliers", "fas_goz"), ("arbitration", "arbitration"), ("security_registries", "fstec"), ("vacancies", "trudvsem"), ], ) def test_branch_ordering_is_boolean_null_last_before_pagination( client, ordering, expected, group, source ): record(group, source, 1, is_branch=True) record(group, source, 2, is_branch=None) record(group, source, 3, is_branch=False) for page, expected_id in enumerate(expected, 1): response = client.get( LIST_URL, {"ordering": ordering, "page_size": 1, "page": page} ) assert response.status_code == 200, response.data assert response.data["data"][0]["external_id"] == str(expected_id) @pytest.mark.parametrize( "group,source,prefix,payloads", [ ( "integrated_structure_documents", "integrated_structures", "first_event_date", [ {"structure": {"first_event_date": date}} for date in ["2026-10-08", "2026-10-09", None] ], ), ( "company_licenses", "checko_licenses", "license_valid_from", [{"valid_from": date} for date in ["2026-10-08", "2026-10-09", None]], ), ( "stock_exchange_activity", "stock_exchange_activity", "traded_on", [ {"exchange": exchange, "trading": {"traded_on": date}} for exchange, date in [ ("moex", "2026-10-08"), ("spb", "2026-10-09"), ("moex", None), ] ], ), ], ) def test_date_ranges_use_business_date_and_include_boundaries( client, group, source, prefix, payloads ): for index, payload in enumerate(payloads, 1): record(group, source, index, payload) response = client.get( LIST_URL, { "source_group": group, f"{prefix}_from": "2026-10-08", f"{prefix}_to": "2026-10-09", "ordering": "uid", }, ) assert response.status_code == 200, response.data assert [item["external_id"] for item in response.data["data"]] == ["1", "2"] for params, code in [ ( {f"{prefix}_from": "2026-10-10", f"{prefix}_to": "2026-10-09"}, "invalid_date_range", ), ({f"{prefix}_from": "09.10.2026"}, "invalid_date"), ( {"source_group": "arbitration", f"{prefix}_from": "2026-10-09"}, "invalid_filter", ), ]: invalid = client.get(LIST_URL, {"source_group": group, **params}) assert invalid.status_code == 400 assert code in [error["code"] for error in invalid.data["errors"]] def test_message_type_options_count_the_same_visible_records(client): record( "leasing", "checko_leasing", 1, {"message_type": "Lease", "message_type_name": "Лизинг"}, ) record( "leasing", "checko_leasing", 2, {"message_type": "Lease", "message_type_name": "Лизинг"}, ) record("leasing", "checko_leasing", 3, {"message_type": "Other"}) record("company_messages", "checko_messages", 4, {"message_type": "Lease"}) response = client.get( LIST_URL + "filter-options/", {"source_group": "leasing", "field": "message_type", "search": "Лиз"}, ) assert response.status_code == 200, response.data assert response.data["data"] == [ {"value": "Lease", "label": "Лизинг", "records_count": 2} ] filtered = client.get( LIST_URL, {"source_group": "leasing", "message_type": "Lease"} ) assert filtered.data["meta"]["pagination"]["total_count"] == 2 assert ( client.get( LIST_URL + "filter-options/", {"source_group": "arbitration", "field": "message_type"}, ).status_code == 400 ) @pytest.mark.parametrize( "source,group,key", [ ("industrial", "industrial_production", "issue_date_normalized"), ("industrial", "industrial_production", "expiry_date_normalized"), ("fstec", "security_registries", "term_date"), ("checko_licenses", "company_licenses", "valid_from"), ], ) def test_normalized_date_ordering_null_last(client, source, group, key): for index, value in enumerate(["2027-01-01", None, "2026-12-31", ""], 1): record(group, source, index, {key: value}) invalid = client.get( LIST_URL, {"source_group": "arbitration", "ordering": "payload__" + key} ) assert invalid.status_code == 400 for ordering, expected in [(key, [3, 1, 2, 4]), ("-" + key, [1, 3, 2, 4])]: response = client.get( LIST_URL, { "source_group": group, "source": source, "ordering": ("-" if ordering.startswith("-") else "") + "payload__" + key, }, ) assert response.status_code == 200, response.data assert [int(item["external_id"]) for item in response.data["data"]] == expected @pytest.mark.parametrize( "raw_key", ["Срок действия сертификата", "Срок действия лицензии", "Срок действия"] ) def test_fstec_ingestion_publishes_normalized_term_without_losing_raw(client, raw_key): organization = Organization.objects.create( name="FSTEC", inn="7707083810", directory_imported_at=timezone.now(), opk_registry_membership=True, ) OrganizationSourceIngestionService.save_records( source="fstec", load_batch=1, records=[ SourceRecordInput( external_id="term", title="Term", organization_name="FSTEC", inn=organization.inn, payload={raw_key: "31.12.2027"}, ) ], ) saved = OrganizationSourceRecord.objects.get() assert saved.payload["term_date"] == "2027-12-31" assert saved.payload["term_raw"] == "31.12.2027" assert saved.payload[raw_key] == "31.12.2027" for path in [LIST_URL, LIST_URL + str(saved.uid) + "/"]: response = client.get(path) assert response.status_code == 200 data = response.data["data"] if path == LIST_URL else response.data item = data[0] if isinstance(data, list) else data assert item["payload"]["term_date"] == "2027-12-31" assert ( _build_record_row(saved, include_financial_lines=False)["payload.term_date"] == "2027-12-31" ) def test_normalization_backfill_is_dry_run_idempotent_and_preserves_identity( client, caplog ): saved = record( "security_registries", "fstec", 1, {"Срок действия": "неизвестен", "filial": ".F."}, filial=".T.", is_branch=None, ) original_updated_at = saved.updated_at output = StringIO() call_command("normalize_testing_amendments", stdout=output) assert json.loads(output.getvalue())["changed_organizations"] == 1 saved.refresh_from_db() assert "term_date" not in saved.payload call_command("normalize_testing_amendments", apply=True, stdout=StringIO()) saved.refresh_from_db() saved.extension.organization.refresh_from_db() assert saved.extension.organization.is_branch is True assert saved.payload["term_date"] is None assert saved.payload["term_raw"] == "неизвестен" assert ( _build_record_row(saved, include_financial_lines=False)[ "organization.is_branch" ] is True ) assert saved.payload["is_branch"] is True assert saved.payload["filial"] == ".F." assert saved.updated_at == original_updated_at assert "invalid_date" in caplog.text assert "branch_conflict" in caplog.text second = StringIO() call_command("normalize_testing_amendments", apply=True, stdout=second) assert json.loads(second.getvalue())["changed_records"] == 0 def test_industrial_ingestion_normalizes_dates_and_logs_invalid_values(client, caplog): organization = Organization.objects.create( name="Industrial", inn="7707083810", directory_imported_at=timezone.now() ) OrganizationSourceIngestionService.save_records( source="industrial", load_batch=1, records=[ SourceRecordInput( external_id="certificate", title="Certificate", organization_name="Industrial", inn=organization.inn, payload={"issue_date": "31.12.2025", "expiry_date": "wrong"}, ) ], ) saved = OrganizationSourceRecord.objects.get() assert saved.payload["issue_date_normalized"] == "2025-12-31" assert saved.payload["expiry_date_normalized"] is None assert saved.payload["expiry_date"] == "wrong" assert "invalid_date" in caplog.text @pytest.mark.parametrize( "group,source", [("government_procurements", "procurements_44fz"), ("arbitration", "arbitration")], ) @pytest.mark.parametrize( "ordering,expected", [("amount", [2, 1, 3]), ("-amount", [1, 2, 3])] ) def test_amount_ordering_remains_decimal_null_last( client, group, source, ordering, expected ): for index, amount in enumerate(["100.00", "20.00", None], 1): saved = record(group, source, index) saved.amount = amount saved.save(update_fields=["amount"]) response = client.get(LIST_URL, {"source_group": group, "ordering": ordering}) assert response.status_code == 200, response.data assert [int(item["external_id"]) for item in response.data["data"]] == expected def test_backfill_batches_repeated_organizations_without_per_record_queries(client): from django.db import connection from django.test.utils import CaptureQueriesContext first = record( "government_procurements", "procurements_44fz", 1, payload={"is_branch": False}, is_branch=False, ) OrganizationSourceRecord.objects.bulk_create( [ OrganizationSourceRecord( extension=first.extension, source=first.source, record_type=first.record_type, external_id=str(index), title="Existing", payload={"is_branch": True}, ) for index in range(2, 1002) ] ) before = list( OrganizationSourceRecord.objects.order_by("uid").values_list( "uid", "updated_at" ) ) with CaptureQueriesContext(connection) as queries: output = StringIO() call_command("normalize_testing_amendments", apply=True, stdout=output) assert len(queries) < 30 assert json.loads(output.getvalue())["scanned"] == 1001 assert not OrganizationSourceRecord.objects.filter(payload__is_branch=True).exists() assert ( list( OrganizationSourceRecord.objects.order_by("uid").values_list( "uid", "updated_at" ) ) == before )