Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 2m43s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Failing after 1m17s
Mostovik Backend CI/CD / Deploy dev (push) Has been skipped
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
433 lines
14 KiB
Python
433 lines
14 KiB
Python
"""Regression cases from the 8 October backend acceptance report."""
|
|
|
|
import json
|
|
from io import StringIO
|
|
from uuid import UUID
|
|
|
|
import pytest
|
|
from django.core.management import call_command
|
|
from django.utils import timezone
|
|
from organizations.models import (
|
|
Organization,
|
|
OrganizationSourceExtension,
|
|
OrganizationSourceRecord,
|
|
)
|
|
from organizations.source_ingestion import (
|
|
OrganizationSourceIngestionService,
|
|
SourceRecordInput,
|
|
)
|
|
from organizations.source_record_export import _build_record_row
|
|
from rest_framework.test import APIClient
|
|
|
|
from tests.apps.user.factories import UserFactory
|
|
|
|
LIST_URL = "/api/v2/organization-source-records/"
|
|
|
|
|
|
@pytest.fixture
|
|
def client(db):
|
|
client = APIClient()
|
|
client.force_authenticate(UserFactory.create_user())
|
|
return client
|
|
|
|
|
|
def record(group, source, index, payload=None, **organization_fields):
|
|
supplied = payload or {}
|
|
if source == "checko_licenses":
|
|
payload = {
|
|
"license_number": "123",
|
|
"issued_on": "2026-01-01",
|
|
"valid_from": None,
|
|
"valid_until": None,
|
|
"authority": "Authority",
|
|
"activities": [],
|
|
"source_url": "https://example.test",
|
|
"fetched_at": "2026-10-09T00:00:00Z",
|
|
"upstream": {},
|
|
**supplied,
|
|
}
|
|
if payload["valid_from"] == "":
|
|
payload["valid_from"] = None
|
|
elif source in ("checko_leasing", "checko_messages"):
|
|
payload = {
|
|
"message_guid": str(index),
|
|
"publication_raw": None,
|
|
"publication_date": None,
|
|
"message_type": "",
|
|
"message_type_name": None,
|
|
"publisher_identifier": None,
|
|
"participant_identifiers": [],
|
|
"source_url": None,
|
|
"fetched_at": "2026-10-09T00:00:00Z",
|
|
"upstream": {},
|
|
"lease_event_type": "test",
|
|
**supplied,
|
|
}
|
|
elif source == "stock_exchange_activity":
|
|
payload = {
|
|
"exchange": "moex",
|
|
"instrument_id": str(index),
|
|
"issuer_inn": "7707083810",
|
|
"issuer_provider_id": None,
|
|
"isin": None,
|
|
"instrument_kind": "stock",
|
|
"listing_date": None,
|
|
"nominal_value": None,
|
|
"nominal_currency": None,
|
|
"parameters": {},
|
|
"duration": None,
|
|
"yield": None,
|
|
"provenance": {},
|
|
"upstream": {},
|
|
**supplied,
|
|
}
|
|
payload["trading"] = {"board_id": "test", "upstream": {}, **payload["trading"]}
|
|
organization = Organization.objects.create(
|
|
name=f"Organization {index}",
|
|
opk_registry_membership=True,
|
|
**organization_fields,
|
|
)
|
|
extension = OrganizationSourceExtension.objects.create(
|
|
organization=organization, source_group=group
|
|
)
|
|
return OrganizationSourceRecord.objects.create(
|
|
uid=UUID(int=index),
|
|
extension=extension,
|
|
source=source,
|
|
record_type="test",
|
|
external_id=str(index),
|
|
payload=payload or {},
|
|
record_date="2026-01-01",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"ordering,expected",
|
|
[
|
|
("organization__is_branch", [3, 1, 2]),
|
|
("-organization__is_branch", [1, 3, 2]),
|
|
],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"group,source",
|
|
[
|
|
("government_procurements", "procurements_44fz"),
|
|
("industrial_production", "industrial"),
|
|
("planned_inspections", "inspections"),
|
|
("defense_suppliers", "fas_goz"),
|
|
("arbitration", "arbitration"),
|
|
("security_registries", "fstec"),
|
|
("vacancies", "trudvsem"),
|
|
],
|
|
)
|
|
def test_branch_ordering_is_boolean_null_last_before_pagination(
|
|
client, ordering, expected, group, source
|
|
):
|
|
record(group, source, 1, is_branch=True)
|
|
record(group, source, 2, is_branch=None)
|
|
record(group, source, 3, is_branch=False)
|
|
for page, expected_id in enumerate(expected, 1):
|
|
response = client.get(
|
|
LIST_URL, {"ordering": ordering, "page_size": 1, "page": page}
|
|
)
|
|
assert response.status_code == 200, response.data
|
|
assert response.data["data"][0]["external_id"] == str(expected_id)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"group,source,prefix,payloads",
|
|
[
|
|
(
|
|
"integrated_structure_documents",
|
|
"integrated_structures",
|
|
"first_event_date",
|
|
[
|
|
{"structure": {"first_event_date": date}}
|
|
for date in ["2026-10-08", "2026-10-09", None]
|
|
],
|
|
),
|
|
(
|
|
"company_licenses",
|
|
"checko_licenses",
|
|
"license_valid_from",
|
|
[{"valid_from": date} for date in ["2026-10-08", "2026-10-09", None]],
|
|
),
|
|
(
|
|
"stock_exchange_activity",
|
|
"stock_exchange_activity",
|
|
"traded_on",
|
|
[
|
|
{"exchange": exchange, "trading": {"traded_on": date}}
|
|
for exchange, date in [
|
|
("moex", "2026-10-08"),
|
|
("spb", "2026-10-09"),
|
|
("moex", None),
|
|
]
|
|
],
|
|
),
|
|
],
|
|
)
|
|
def test_date_ranges_use_business_date_and_include_boundaries(
|
|
client, group, source, prefix, payloads
|
|
):
|
|
for index, payload in enumerate(payloads, 1):
|
|
record(group, source, index, payload)
|
|
response = client.get(
|
|
LIST_URL,
|
|
{
|
|
"source_group": group,
|
|
f"{prefix}_from": "2026-10-08",
|
|
f"{prefix}_to": "2026-10-09",
|
|
"ordering": "uid",
|
|
},
|
|
)
|
|
assert response.status_code == 200, response.data
|
|
assert [item["external_id"] for item in response.data["data"]] == ["1", "2"]
|
|
for params, code in [
|
|
(
|
|
{f"{prefix}_from": "2026-10-10", f"{prefix}_to": "2026-10-09"},
|
|
"invalid_date_range",
|
|
),
|
|
({f"{prefix}_from": "09.10.2026"}, "invalid_date"),
|
|
(
|
|
{"source_group": "arbitration", f"{prefix}_from": "2026-10-09"},
|
|
"invalid_filter",
|
|
),
|
|
]:
|
|
invalid = client.get(LIST_URL, {"source_group": group, **params})
|
|
assert invalid.status_code == 400
|
|
assert code in [error["code"] for error in invalid.data["errors"]]
|
|
|
|
|
|
def test_message_type_options_count_the_same_visible_records(client):
|
|
record(
|
|
"leasing",
|
|
"checko_leasing",
|
|
1,
|
|
{"message_type": "Lease", "message_type_name": "Лизинг"},
|
|
)
|
|
record(
|
|
"leasing",
|
|
"checko_leasing",
|
|
2,
|
|
{"message_type": "Lease", "message_type_name": "Лизинг"},
|
|
)
|
|
record("leasing", "checko_leasing", 3, {"message_type": "Other"})
|
|
record("company_messages", "checko_messages", 4, {"message_type": "Lease"})
|
|
response = client.get(
|
|
LIST_URL + "filter-options/",
|
|
{"source_group": "leasing", "field": "message_type", "search": "Лиз"},
|
|
)
|
|
assert response.status_code == 200, response.data
|
|
assert response.data["data"] == [
|
|
{"value": "Lease", "label": "Лизинг", "records_count": 2}
|
|
]
|
|
filtered = client.get(
|
|
LIST_URL, {"source_group": "leasing", "message_type": "Lease"}
|
|
)
|
|
assert filtered.data["meta"]["pagination"]["total_count"] == 2
|
|
assert (
|
|
client.get(
|
|
LIST_URL + "filter-options/",
|
|
{"source_group": "arbitration", "field": "message_type"},
|
|
).status_code
|
|
== 400
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"source,group,key",
|
|
[
|
|
("industrial", "industrial_production", "issue_date_normalized"),
|
|
("industrial", "industrial_production", "expiry_date_normalized"),
|
|
("fstec", "security_registries", "term_date"),
|
|
("checko_licenses", "company_licenses", "valid_from"),
|
|
],
|
|
)
|
|
def test_normalized_date_ordering_null_last(client, source, group, key):
|
|
for index, value in enumerate(["2027-01-01", None, "2026-12-31", ""], 1):
|
|
record(group, source, index, {key: value})
|
|
invalid = client.get(
|
|
LIST_URL, {"source_group": "arbitration", "ordering": "payload__" + key}
|
|
)
|
|
assert invalid.status_code == 400
|
|
for ordering, expected in [(key, [3, 1, 2, 4]), ("-" + key, [1, 3, 2, 4])]:
|
|
response = client.get(
|
|
LIST_URL,
|
|
{
|
|
"source_group": group,
|
|
"source": source,
|
|
"ordering": ("-" if ordering.startswith("-") else "")
|
|
+ "payload__"
|
|
+ key,
|
|
},
|
|
)
|
|
assert response.status_code == 200, response.data
|
|
assert [int(item["external_id"]) for item in response.data["data"]] == expected
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw_key", ["Срок действия сертификата", "Срок действия лицензии", "Срок действия"]
|
|
)
|
|
def test_fstec_ingestion_publishes_normalized_term_without_losing_raw(client, raw_key):
|
|
organization = Organization.objects.create(
|
|
name="FSTEC",
|
|
inn="7707083810",
|
|
directory_imported_at=timezone.now(),
|
|
opk_registry_membership=True,
|
|
)
|
|
OrganizationSourceIngestionService.save_records(
|
|
source="fstec",
|
|
load_batch=1,
|
|
records=[
|
|
SourceRecordInput(
|
|
external_id="term",
|
|
title="Term",
|
|
organization_name="FSTEC",
|
|
inn=organization.inn,
|
|
payload={raw_key: "31.12.2027"},
|
|
)
|
|
],
|
|
)
|
|
saved = OrganizationSourceRecord.objects.get()
|
|
assert saved.payload["term_date"] == "2027-12-31"
|
|
assert saved.payload["term_raw"] == "31.12.2027"
|
|
assert saved.payload[raw_key] == "31.12.2027"
|
|
for path in [LIST_URL, LIST_URL + str(saved.uid) + "/"]:
|
|
response = client.get(path)
|
|
assert response.status_code == 200
|
|
data = response.data["data"] if path == LIST_URL else response.data
|
|
item = data[0] if isinstance(data, list) else data
|
|
assert item["payload"]["term_date"] == "2027-12-31"
|
|
assert (
|
|
_build_record_row(saved, include_financial_lines=False)["payload.term_date"]
|
|
== "2027-12-31"
|
|
)
|
|
|
|
|
|
def test_normalization_backfill_is_dry_run_idempotent_and_preserves_identity(
|
|
client, caplog
|
|
):
|
|
saved = record(
|
|
"security_registries",
|
|
"fstec",
|
|
1,
|
|
{"Срок действия": "неизвестен", "filial": ".F."},
|
|
filial=".T.",
|
|
is_branch=None,
|
|
)
|
|
original_updated_at = saved.updated_at
|
|
output = StringIO()
|
|
call_command("normalize_testing_amendments", stdout=output)
|
|
assert json.loads(output.getvalue())["changed_organizations"] == 1
|
|
saved.refresh_from_db()
|
|
assert "term_date" not in saved.payload
|
|
call_command("normalize_testing_amendments", apply=True, stdout=StringIO())
|
|
saved.refresh_from_db()
|
|
saved.extension.organization.refresh_from_db()
|
|
assert saved.extension.organization.is_branch is True
|
|
assert saved.payload["term_date"] is None
|
|
assert saved.payload["term_raw"] == "неизвестен"
|
|
assert (
|
|
_build_record_row(saved, include_financial_lines=False)[
|
|
"organization.is_branch"
|
|
]
|
|
is True
|
|
)
|
|
assert saved.payload["is_branch"] is True
|
|
assert saved.payload["filial"] == ".F."
|
|
assert saved.updated_at == original_updated_at
|
|
assert "invalid_date" in caplog.text
|
|
assert "branch_conflict" in caplog.text
|
|
second = StringIO()
|
|
call_command("normalize_testing_amendments", apply=True, stdout=second)
|
|
assert json.loads(second.getvalue())["changed_records"] == 0
|
|
|
|
|
|
def test_industrial_ingestion_normalizes_dates_and_logs_invalid_values(client, caplog):
|
|
organization = Organization.objects.create(
|
|
name="Industrial", inn="7707083810", directory_imported_at=timezone.now()
|
|
)
|
|
OrganizationSourceIngestionService.save_records(
|
|
source="industrial",
|
|
load_batch=1,
|
|
records=[
|
|
SourceRecordInput(
|
|
external_id="certificate",
|
|
title="Certificate",
|
|
organization_name="Industrial",
|
|
inn=organization.inn,
|
|
payload={"issue_date": "31.12.2025", "expiry_date": "wrong"},
|
|
)
|
|
],
|
|
)
|
|
saved = OrganizationSourceRecord.objects.get()
|
|
assert saved.payload["issue_date_normalized"] == "2025-12-31"
|
|
assert saved.payload["expiry_date_normalized"] is None
|
|
assert saved.payload["expiry_date"] == "wrong"
|
|
assert "invalid_date" in caplog.text
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"group,source",
|
|
[("government_procurements", "procurements_44fz"), ("arbitration", "arbitration")],
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"ordering,expected", [("amount", [2, 1, 3]), ("-amount", [1, 2, 3])]
|
|
)
|
|
def test_amount_ordering_remains_decimal_null_last(
|
|
client, group, source, ordering, expected
|
|
):
|
|
for index, amount in enumerate(["100.00", "20.00", None], 1):
|
|
saved = record(group, source, index)
|
|
saved.amount = amount
|
|
saved.save(update_fields=["amount"])
|
|
response = client.get(LIST_URL, {"source_group": group, "ordering": ordering})
|
|
assert response.status_code == 200, response.data
|
|
assert [int(item["external_id"]) for item in response.data["data"]] == expected
|
|
|
|
|
|
def test_backfill_batches_repeated_organizations_without_per_record_queries(client):
|
|
from django.db import connection
|
|
from django.test.utils import CaptureQueriesContext
|
|
|
|
first = record(
|
|
"government_procurements",
|
|
"procurements_44fz",
|
|
1,
|
|
payload={"is_branch": False},
|
|
is_branch=False,
|
|
)
|
|
OrganizationSourceRecord.objects.bulk_create(
|
|
[
|
|
OrganizationSourceRecord(
|
|
extension=first.extension,
|
|
source=first.source,
|
|
record_type=first.record_type,
|
|
external_id=str(index),
|
|
title="Existing",
|
|
payload={"is_branch": True},
|
|
)
|
|
for index in range(2, 1002)
|
|
]
|
|
)
|
|
before = list(
|
|
OrganizationSourceRecord.objects.order_by("uid").values_list(
|
|
"uid", "updated_at"
|
|
)
|
|
)
|
|
with CaptureQueriesContext(connection) as queries:
|
|
output = StringIO()
|
|
call_command("normalize_testing_amendments", apply=True, stdout=output)
|
|
assert len(queries) < 30
|
|
assert json.loads(output.getvalue())["scanned"] == 1001
|
|
assert not OrganizationSourceRecord.objects.filter(payload__is_branch=True).exists()
|
|
assert (
|
|
list(
|
|
OrganizationSourceRecord.objects.order_by("uid").values_list(
|
|
"uid", "updated_at"
|
|
)
|
|
)
|
|
== before
|
|
)
|