18971d33ec
All checks were successful
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m55s
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Successful in 3m43s
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been skipped
Mostovik Backend CI/CD / Deploy customer main (push) Has been skipped
Mostovik Backend CI/CD / Deploy dev (push) Successful in 1m45s
366 lines
14 KiB
Python
366 lines
14 KiB
Python
"""Public source identity, published counts, provenance and pollable status contracts."""
|
|
|
|
import csv
|
|
from datetime import date
|
|
from importlib import import_module
|
|
from io import StringIO
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
from apps.core.models import BackgroundJob
|
|
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
|
|
from apps.parsers.source_cards import SourceCardService
|
|
from core.celery import app as celery_app
|
|
from django.core.cache import cache
|
|
from django.urls import reverse
|
|
from django.utils import timezone
|
|
from organizations.models import (
|
|
Organization,
|
|
OrganizationSourceExtension,
|
|
OrganizationSourceRecord,
|
|
)
|
|
from rest_framework.test import APIClient
|
|
|
|
from tests.apps.user.factories import UserFactory
|
|
|
|
SOURCES = (
|
|
(
|
|
"budget-process-registry",
|
|
"budget_ubpandnubp",
|
|
"budget_process_registry",
|
|
"budget_registry_organization",
|
|
),
|
|
(
|
|
"sme-support-recipients-registry",
|
|
"fns_sme_support_recipients",
|
|
"government_support",
|
|
"sme_support_measure",
|
|
),
|
|
(
|
|
"sro-membership-check",
|
|
"sro_membership_check",
|
|
"sro_membership",
|
|
"sro_membership",
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def admin_client(db):
|
|
cache.clear()
|
|
client = APIClient()
|
|
client.force_authenticate(UserFactory.create_user(is_staff=True))
|
|
return client
|
|
|
|
|
|
@pytest.mark.parametrize("slug,source,group,record_type", SOURCES)
|
|
def test_cards_dashboard_and_history_share_published_scope_and_provenance(
|
|
admin_client, slug, source, group, record_type
|
|
):
|
|
log = ParserLoadLog.objects.create(
|
|
source=source, batch_id=1, records_count=3, status="success"
|
|
)
|
|
artifact = ParserSourceArtifact.objects.create(
|
|
source=source,
|
|
load_batch=1,
|
|
status="published",
|
|
source_published_at=date(2026, 9, 1),
|
|
version="fixture-v1",
|
|
sha256="a" * 64,
|
|
parsed_count=4,
|
|
published_count=3,
|
|
quarantined_count=1,
|
|
)
|
|
for index, is_opk in enumerate((True, False)):
|
|
org = Organization.objects.create(
|
|
name=f"Fixture {index}",
|
|
inn=f"000000000{index}",
|
|
ogrn=f"000000000000{index}",
|
|
okpo=f"0000000{index}",
|
|
directory_imported_at=timezone.now(),
|
|
opk_registry_membership=is_opk,
|
|
)
|
|
extension = OrganizationSourceExtension.objects.create(
|
|
organization=org, source_group=group, records_count=2 if is_opk else 1
|
|
)
|
|
for number in range(2 if is_opk else 1):
|
|
OrganizationSourceRecord.objects.create(
|
|
extension=extension,
|
|
source=source,
|
|
record_type=record_type,
|
|
external_id=f"{index}-{number}",
|
|
)
|
|
ParserStagedRecord.objects.create(
|
|
artifact=artifact,
|
|
organization=org,
|
|
disposition="published",
|
|
row_number=index * 2 + number + 1,
|
|
)
|
|
SourceCardService.clear_cache()
|
|
detail = admin_client.get(f"/api/v1/sources/{slug}/").data["data"]
|
|
cards = admin_client.get("/api/v1/sources/").data["data"]
|
|
assert next(card for card in cards if card["slug"] == slug) == detail
|
|
assert (detail["records_count"], detail["organizations_count"]) == (3, 2)
|
|
assert detail["source_items"][0]["records_count"] == 3
|
|
assert detail["active_tasks"] == []
|
|
expected = detail["latest_success_load"]["snapshot"]
|
|
assert expected["artifact_id"] == str(artifact.pk)
|
|
assert expected["snapshot_date"] == "2026-09-01"
|
|
assert expected["checksum_sha256"] == "a" * 64
|
|
assert expected["published_at"]
|
|
dashboard = admin_client.get("/api/v1/parsers/dashboard/").data["data"]
|
|
assert dashboard["source_counts"][source] == 3
|
|
assert next(item for item in dashboard["sources"] if item["source"] == source)
|
|
dashboard_log = next(
|
|
item for item in dashboard["load_logs"] if item["id"] == log.pk
|
|
)
|
|
assert dashboard_log["snapshot"] == expected
|
|
logs = admin_client.get(
|
|
reverse("api_v1:system:parser-logs-list"), {"source": source}
|
|
).data["results"]
|
|
assert len(logs) == 1
|
|
assert (
|
|
logs[0]["source"],
|
|
logs[0]["records_count"],
|
|
logs[0]["organizations_count"],
|
|
) == (source, 3, 2)
|
|
assert logs[0]["snapshot"] == expected
|
|
response = admin_client.get(
|
|
reverse("api_v1:system:parser-logs-detail", kwargs={"pk": log.pk})
|
|
)
|
|
assert response.data["snapshot"] == expected
|
|
assert response.data["source_label"]
|
|
exported = admin_client.get(
|
|
reverse("api_v1:system:parser-logs-export"), {"source": source}
|
|
)
|
|
csv_rows = list(
|
|
csv.DictReader(StringIO(exported.content.decode("utf-8-sig")), delimiter=";")
|
|
)
|
|
assert csv_rows[0]["Код parser source"] == source
|
|
assert csv_rows[0]["SHA256 снимка"] == "a" * 64
|
|
# A failed attempt remains visible without replacing the last good snapshot.
|
|
failed = ParserLoadLog.objects.create(
|
|
source=source, batch_id=2, status="failed", error_message="fixture failure"
|
|
)
|
|
card = admin_client.get(f"/api/v1/sources/{slug}/").data["data"]
|
|
assert card["latest_load"]["batch_id"] == failed.batch_id
|
|
assert card["latest_success_load"]["snapshot"] == expected
|
|
assert card["last_updated_at"] == detail["last_updated_at"]
|
|
assert (card["records_count"], card["organizations_count"]) == (3, 2)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"url",
|
|
(
|
|
"/api/v1/sources/sro-membership-check/refresh/",
|
|
"/api/v1/parsers/run/sro_membership_check/",
|
|
),
|
|
)
|
|
def test_sro_default_gate_is_typed_and_does_not_enqueue(admin_client, settings, url):
|
|
settings.SRO_UPSTREAM_ACCESS_APPROVED = False
|
|
settings.SRO_UPSTREAM_APPROVAL_REFERENCE = ""
|
|
with patch(
|
|
"apps.parsers.tasks_registry_snapshots.parse_sro_membership.apply_async"
|
|
) as dispatch:
|
|
response = admin_client.post(url, {}, format="json")
|
|
assert response.status_code == 409
|
|
assert response.data["errors"][0]["code"] == "upstream_access_not_approved"
|
|
assert not BackgroundJob.objects.exists()
|
|
dispatch.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"first",
|
|
(
|
|
"/api/v1/sources/sro-membership-check/refresh/",
|
|
"/api/v1/parsers/run/sro_membership_check/",
|
|
),
|
|
)
|
|
def test_approved_sro_refresh_polling_and_cross_entrypoint_conflict(
|
|
admin_client, settings, first
|
|
):
|
|
settings.SRO_UPSTREAM_ACCESS_APPROVED = True
|
|
settings.SRO_UPSTREAM_APPROVAL_REFERENCE = "fixture approval"
|
|
import_module("apps.parsers.tasks")
|
|
task = celery_app.tasks["parsers.sro_membership_check.refresh"]
|
|
with patch.object(task, "apply_async") as dispatch:
|
|
response = admin_client.post(first, {"params": {}}, format="json")
|
|
assert response.status_code == 202
|
|
payload = response.data.get("data", response.data)
|
|
assert payload["status"] == "queued"
|
|
assert payload["task_ids"] == [payload["task_id"]]
|
|
job = admin_client.get(f"/api/v1/jobs/{payload['task_id']}/").data
|
|
assert job["source"] == "sro_membership_check"
|
|
assert job["status"] == "queued"
|
|
other = (
|
|
"/api/v1/parsers/run/sro_membership_check/"
|
|
if "/sources/" in first
|
|
else "/api/v1/sources/sro-membership-check/refresh/"
|
|
)
|
|
conflict = admin_client.post(other, {}, format="json")
|
|
assert conflict.status_code == 409
|
|
assert conflict.data["errors"][0]["code"] == "refresh_already_running"
|
|
assert dispatch.call_count == 1
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw,expected",
|
|
(
|
|
("pending", "queued"),
|
|
("started", "running"),
|
|
("retry", "retrying"),
|
|
("success", "success"),
|
|
("failure", "failed"),
|
|
("revoked", "cancelled"),
|
|
),
|
|
)
|
|
def test_registry_job_status_distinguishes_queue_retry_failure_cancel(
|
|
admin_client, raw, expected
|
|
):
|
|
job = BackgroundJob.objects.create(
|
|
task_id="fixture-status",
|
|
task_name="parsers.budget_ubpandnubp.refresh",
|
|
status=raw,
|
|
progress=47,
|
|
progress_message="fixture progress",
|
|
)
|
|
response = admin_client.get(f"/api/v1/jobs/{job.task_id}/")
|
|
assert response.status_code == 200
|
|
assert response.data["status"] == expected
|
|
assert response.data["progress"] == 47
|
|
assert response.data["message"] == "fixture progress"
|
|
if raw == "success":
|
|
job.result = {"status": "skipped"}
|
|
job.save(update_fields=["result"])
|
|
assert (
|
|
admin_client.get(f"/api/v1/jobs/{job.task_id}/").data["status"] == "skipped"
|
|
)
|
|
|
|
|
|
def test_typed_refresh_job_dashboard_history_openapi(admin_client):
|
|
response = admin_client.get(reverse("schema-swagger-ui"), {"format": "openapi"})
|
|
assert response.status_code == 200
|
|
definitions = response.data["definitions"]
|
|
properties = definitions["SourceCard"]["properties"]
|
|
assert {
|
|
"source_items",
|
|
"latest_load",
|
|
"latest_success_load",
|
|
"active_tasks",
|
|
} <= properties.keys()
|
|
conflict = definitions["SourceRefreshConflictItem"]["properties"]["code"]["enum"]
|
|
assert set(conflict) == {"refresh_already_running", "upstream_access_not_approved"}
|
|
assert "SnapshotMetadata" in definitions
|
|
assert "SnapshotJobResult" in definitions
|
|
assert "ParserDashboardResponse" in definitions
|
|
assert "cancelled" in definitions["BackgroundJob"]["properties"]["status"]["enum"]
|
|
paths = response.data["paths"]
|
|
dashboard = next(path for path in paths if path.endswith("/parsers/dashboard/"))
|
|
assert paths[dashboard]["get"]["responses"]["200"]["schema"]["$ref"].endswith(
|
|
"/ParserDashboardResponse"
|
|
)
|
|
assert paths[dashboard]["get"]["operationId"] == "api_v1_parsers_dashboard_list"
|
|
assert paths[dashboard]["get"]["tags"] == ["api"]
|
|
assert {"pending", "started", "retry", "failure", "revoked"} <= set(
|
|
definitions["BackgroundJob"]["properties"]["status"]["enum"]
|
|
)
|
|
logs_path = next(path for path in paths if path.endswith("/system/logs/"))
|
|
parameters = {item["name"]: item for item in paths[logs_path]["get"]["parameters"]}
|
|
assert parameters["source"]["type"] == "string"
|
|
assert "alias" in parameters["source"]["description"]
|
|
assert parameters["batch_id"]["type"] == "integer"
|
|
results_list = next(
|
|
path for path in paths if path.endswith("/parsers/results/{source_key}/")
|
|
)
|
|
assert paths[results_list]["get"]["operationId"] == "api_v1_parsers_results_list"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"body,expected_mode",
|
|
(
|
|
({}, "incremental"),
|
|
({"mode": "incremental"}, "incremental"),
|
|
({"mode": "full"}, "full"),
|
|
),
|
|
)
|
|
def test_sro_parser_run_accepts_explicit_or_default_mode(
|
|
admin_client, settings, body, expected_mode
|
|
):
|
|
settings.SRO_UPSTREAM_ACCESS_APPROVED = True
|
|
settings.SRO_UPSTREAM_APPROVAL_REFERENCE = "fixture approval"
|
|
with patch(
|
|
"apps.parsers.tasks_registry_snapshots.parse_sro_membership.apply_async"
|
|
) as dispatch:
|
|
response = admin_client.post(
|
|
"/api/v1/parsers/run/sro_membership_check/", body, format="json"
|
|
)
|
|
assert response.status_code == 202
|
|
assert dispatch.call_args.kwargs["kwargs"]["mode"] == expected_mode
|
|
|
|
|
|
@pytest.mark.parametrize("mode", ("unknown", "", None, 1, []))
|
|
def test_sro_invalid_mode_is_typed_and_never_enqueues(admin_client, mode):
|
|
with patch(
|
|
"apps.parsers.tasks_registry_snapshots.parse_sro_membership.apply_async"
|
|
) as dispatch:
|
|
response = admin_client.post(
|
|
"/api/v1/parsers/run/sro_membership_check/", {"mode": mode}, format="json"
|
|
)
|
|
assert response.status_code == 400
|
|
assert response.data["errors"][0]["code"] == "invalid_mode"
|
|
dispatch.assert_not_called()
|
|
|
|
|
|
def test_sro_job_and_history_expose_typed_actual_counters_only(admin_client):
|
|
metadata = {
|
|
"mode": "incremental",
|
|
"candidate_organizations_count": 7,
|
|
"queried_organizations_count": 3,
|
|
"found_memberships_count": 2,
|
|
"snapshot_records_count": 10,
|
|
"snapshot_organizations_count": 8,
|
|
"http_errors_count": 1,
|
|
"parse_errors_count": 2,
|
|
"upstream_registry_date": None,
|
|
"private_loader_setting": "must not expose",
|
|
}
|
|
job = BackgroundJob.objects.create(
|
|
task_id="metadata",
|
|
task_name="parsers.sro_membership_check.refresh",
|
|
status="started",
|
|
meta=metadata,
|
|
)
|
|
payload = admin_client.get(f"/api/v1/jobs/{job.task_id}/").data
|
|
assert payload["meta"]["candidate_organizations_count"] == 7
|
|
assert payload["meta"]["source"] == "sro_membership_check"
|
|
assert payload["meta"]["upstream_registry_date"] is None
|
|
assert "private_loader_setting" not in payload["meta"]
|
|
log = ParserLoadLog.objects.create(
|
|
source="sro_membership_check", batch_id=1, status="success", records_count=10
|
|
)
|
|
ParserSourceArtifact.objects.create(
|
|
source="sro_membership_check",
|
|
load_batch=1,
|
|
status="published",
|
|
published_count=10,
|
|
metadata=metadata,
|
|
)
|
|
response = admin_client.get(
|
|
reverse("api_v1:system:parser-logs-detail", kwargs={"pk": log.pk})
|
|
)
|
|
assert response.data["organizations_count"] == 8
|
|
assert response.data["meta"]["http_errors_count"] == 1
|
|
assert response.data["meta"]["parse_errors_count"] == 2
|
|
assert "private_loader_setting" not in response.data["meta"]
|
|
schema = admin_client.get(reverse("schema-swagger-ui"), {"format": "openapi"}).data
|
|
assert (
|
|
schema["definitions"]["SnapshotRunMetadata"]["properties"][
|
|
"queried_organizations_count"
|
|
]["type"]
|
|
== "integer"
|
|
)
|
|
assert (
|
|
"organizations_count"
|
|
in schema["definitions"]["SnapshotJobResult"]["properties"]
|
|
)
|