Files
mostovik-backend/tests/apps/parsers/test_snapshot_public_contract.py
Aleksandr Meshchryakov 9b3da5b505
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m16s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
fix: implement backend testing amendments and atomic source publication
2026-10-12 09:47:28 +02:00

397 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Public source identity, published counts, provenance and pollable status contracts."""
import csv
from datetime import date
from importlib import import_module
from io import StringIO
from unittest.mock import patch
import pytest
from apps.core.models import BackgroundJob
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
from apps.parsers.source_cards import SourceCardService
from core.celery import app as celery_app
from django.core.cache import cache
from django.urls import reverse
from django.utils import timezone
from organizations.models import (
Organization,
OrganizationSourceExtension,
OrganizationSourceRecord,
)
from rest_framework.test import APIClient
from tests.apps.user.factories import UserFactory
SOURCES = (
(
"fssp-enforcement-proceedings",
"fssp_enforcement_proceedings",
"enforcement_proceedings",
"enforcement_proceeding",
),
(
"budget-process-registry",
"budget_ubpandnubp",
"budget_process_registry",
"budget_registry_organization",
),
(
"sme-support-recipients-registry",
"fns_sme_support_recipients",
"government_support",
"sme_support_measure",
),
(
"sro-membership-check",
"sro_membership_check",
"sro_membership",
"sro_membership",
),
)
@pytest.fixture
def admin_client(db):
cache.clear()
client = APIClient()
client.force_authenticate(UserFactory.create_user(is_staff=True))
return client
@pytest.mark.parametrize("slug,source,group,record_type", SOURCES)
def test_cards_dashboard_and_history_share_published_scope_and_provenance(
admin_client, slug, source, group, record_type
):
log = ParserLoadLog.objects.create(
source=source, batch_id=1, records_count=3, status="success"
)
artifact = ParserSourceArtifact.objects.create(
source=source,
load_batch=1,
status="published",
source_published_at=date(2026, 9, 1),
version="fixture-v1",
sha256="a" * 64,
parsed_count=4,
published_count=3,
quarantined_count=1,
)
for index, is_opk in enumerate((True, False)):
org = Organization.objects.create(
name=f"Fixture {index}",
inn=f"000000000{index}",
ogrn=f"000000000000{index}",
okpo=f"0000000{index}",
directory_imported_at=timezone.now(),
opk_registry_membership=is_opk,
)
extension = OrganizationSourceExtension.objects.create(
organization=org, source_group=group, records_count=2 if is_opk else 1
)
for number in range(2 if is_opk else 1):
OrganizationSourceRecord.objects.create(
extension=extension,
source=source,
record_type=record_type,
external_id=f"{index}-{number}",
)
ParserStagedRecord.objects.create(
artifact=artifact,
organization=org,
disposition="published",
row_number=index * 2 + number + 1,
)
SourceCardService.clear_cache()
detail = admin_client.get(f"/api/v1/sources/{slug}/").data["data"]
cards = admin_client.get("/api/v1/sources/").data["data"]
assert next(card for card in cards if card["slug"] == slug) == detail
assert (detail["records_count"], detail["organizations_count"]) == (3, 2)
assert detail["source_items"][0]["records_count"] == 3
assert detail["active_tasks"] == []
expected = detail["latest_success_load"]["snapshot"]
assert expected["artifact_id"] == str(artifact.pk)
assert expected["snapshot_date"] == "2026-09-01"
assert expected["checksum_sha256"] == "a" * 64
assert expected["published_at"]
dashboard = admin_client.get("/api/v1/parsers/dashboard/").data["data"]
assert dashboard["source_counts"][source] == 3
assert next(item for item in dashboard["sources"] if item["source"] == source)
dashboard_log = next(
item for item in dashboard["load_logs"] if item["id"] == log.pk
)
assert dashboard_log["snapshot"] == expected
logs = admin_client.get(
reverse("api_v1:system:parser-logs-list"), {"source": source}
).data["results"]
assert len(logs) == 1
assert (
logs[0]["source"],
logs[0]["records_count"],
logs[0]["organizations_count"],
) == (source, 3, 2)
assert logs[0]["snapshot"] == expected
response = admin_client.get(
reverse("api_v1:system:parser-logs-detail", kwargs={"pk": log.pk})
)
assert response.data["snapshot"] == expected
assert response.data["source_label"]
exported = admin_client.get(
reverse("api_v1:system:parser-logs-export"), {"source": source}
)
csv_rows = list(
csv.DictReader(StringIO(exported.content.decode("utf-8-sig")), delimiter=";")
)
assert csv_rows[0]["Код parser source"] == source
assert csv_rows[0]["SHA256 снимка"] == "a" * 64
# A failed attempt remains visible without replacing the last good snapshot.
failed = ParserLoadLog.objects.create(
source=source, batch_id=2, status="failed", error_message="fixture failure"
)
card = admin_client.get(f"/api/v1/sources/{slug}/").data["data"]
assert card["latest_load"]["batch_id"] == failed.batch_id
assert card["latest_success_load"]["snapshot"] == expected
assert card["last_updated_at"] == detail["last_updated_at"]
assert (card["records_count"], card["organizations_count"]) == (3, 2)
@pytest.mark.parametrize(
"url",
(
"/api/v1/sources/sro-membership-check/refresh/",
"/api/v1/parsers/run/sro_membership_check/",
),
)
def test_sro_default_gate_is_typed_and_does_not_enqueue(admin_client, settings, url):
settings.SRO_UPSTREAM_ACCESS_APPROVED = False
settings.SRO_UPSTREAM_APPROVAL_REFERENCE = ""
with patch(
"apps.parsers.tasks_registry_snapshots.parse_sro_membership.apply_async"
) as dispatch:
response = admin_client.post(url, {}, format="json")
assert response.status_code == 409
assert response.data["errors"][0]["code"] == "upstream_access_not_approved"
assert not BackgroundJob.objects.exists()
dispatch.assert_not_called()
@pytest.mark.parametrize(
"first",
(
"/api/v1/sources/sro-membership-check/refresh/",
"/api/v1/parsers/run/sro_membership_check/",
),
)
def test_approved_sro_refresh_polling_and_cross_entrypoint_conflict(
admin_client, settings, first
):
settings.SRO_UPSTREAM_ACCESS_APPROVED = True
settings.SRO_UPSTREAM_APPROVAL_REFERENCE = "fixture approval"
import_module("apps.parsers.tasks")
task = celery_app.tasks["parsers.sro_membership_check.refresh"]
with patch.object(task, "apply_async") as dispatch:
response = admin_client.post(first, {"params": {}}, format="json")
assert response.status_code == 202
payload = response.data.get("data", response.data)
assert payload["status"] == "queued"
assert payload["task_ids"] == [payload["task_id"]]
job = admin_client.get(f"/api/v1/jobs/{payload['task_id']}/").data
assert job["source"] == "sro_membership_check"
assert job["status"] == "queued"
other = (
"/api/v1/parsers/run/sro_membership_check/"
if "/sources/" in first
else "/api/v1/sources/sro-membership-check/refresh/"
)
conflict = admin_client.post(other, {}, format="json")
assert conflict.status_code == 409
assert conflict.data["errors"][0]["code"] == "refresh_already_running"
assert dispatch.call_count == 1
@pytest.mark.parametrize(
"raw,expected",
(
("pending", "queued"),
("started", "running"),
("retry", "retrying"),
("success", "success"),
("failure", "failed"),
("revoked", "cancelled"),
),
)
def test_registry_job_status_distinguishes_queue_retry_failure_cancel(
admin_client, raw, expected
):
job = BackgroundJob.objects.create(
task_id="fixture-status",
task_name="parsers.budget_ubpandnubp.refresh",
status=raw,
progress=47,
progress_message="fixture progress",
)
response = admin_client.get(f"/api/v1/jobs/{job.task_id}/")
assert response.status_code == 200
assert response.data["status"] == expected
assert response.data["progress"] == 47
assert response.data["message"] == (
"Не удалось обновить данные. Повторите попытку позже."
if expected in {"failed", "cancelled"}
else "fixture progress"
)
if raw == "success":
job.result = {"status": "skipped"}
job.save(update_fields=["result"])
assert (
admin_client.get(f"/api/v1/jobs/{job.task_id}/").data["status"] == "skipped"
)
def test_typed_refresh_job_dashboard_history_openapi(admin_client):
response = admin_client.get(reverse("schema-swagger-ui"), {"format": "openapi"})
assert response.status_code == 200
definitions = response.data["definitions"]
properties = definitions["SourceCard"]["properties"]
assert {
"source_items",
"latest_load",
"latest_success_load",
"active_tasks",
} <= properties.keys()
conflict = definitions["SourceRefreshConflictItem"]["properties"]["code"]["enum"]
assert set(conflict) == {"refresh_already_running", "upstream_access_not_approved"}
assert "SnapshotMetadata" in definitions
assert "SnapshotJobResult" in definitions
assert "ParserDashboardResponse" in definitions
assert "cancelled" in definitions["BackgroundJob"]["properties"]["status"]["enum"]
paths = response.data["paths"]
dashboard = next(path for path in paths if path.endswith("/parsers/dashboard/"))
assert paths[dashboard]["get"]["responses"]["200"]["schema"]["$ref"].endswith(
"/ParserDashboardResponse"
)
assert paths[dashboard]["get"]["operationId"] == "api_v1_parsers_dashboard_list"
assert paths[dashboard]["get"]["tags"] == ["api"]
assert {
"queued",
"running",
"retrying",
"failed",
"cancelled",
"success",
"skipped",
} == set(definitions["BackgroundJob"]["properties"]["status"]["enum"])
logs_path = next(path for path in paths if path.endswith("/system/logs/"))
parameters = {item["name"]: item for item in paths[logs_path]["get"]["parameters"]}
assert parameters["source"]["type"] == "string"
assert "alias" in parameters["source"]["description"]
assert parameters["batch_id"]["type"] == "integer"
results_list = next(
path for path in paths if path.endswith("/parsers/results/{source_key}/")
)
assert paths[results_list]["get"]["operationId"] == "api_v1_parsers_results_list"
@pytest.mark.parametrize(
"body,expected_mode",
(
({}, "incremental"),
({"mode": "incremental"}, "incremental"),
({"mode": "full"}, "full"),
),
)
def test_sro_parser_run_accepts_explicit_or_default_mode(
admin_client, settings, body, expected_mode
):
settings.SRO_UPSTREAM_ACCESS_APPROVED = True
settings.SRO_UPSTREAM_APPROVAL_REFERENCE = "fixture approval"
with patch(
"apps.parsers.tasks_registry_snapshots.parse_sro_membership.apply_async"
) as dispatch:
response = admin_client.post(
"/api/v1/parsers/run/sro_membership_check/", body, format="json"
)
assert response.status_code == 202
assert dispatch.call_args.kwargs["kwargs"]["mode"] == expected_mode
@pytest.mark.parametrize("mode", ("unknown", "", None, 1, []))
def test_sro_invalid_mode_is_typed_and_never_enqueues(admin_client, mode):
with patch(
"apps.parsers.tasks_registry_snapshots.parse_sro_membership.apply_async"
) as dispatch:
response = admin_client.post(
"/api/v1/parsers/run/sro_membership_check/", {"mode": mode}, format="json"
)
assert response.status_code == 400
assert response.data["errors"][0]["code"] == "invalid_mode"
dispatch.assert_not_called()
def test_sro_job_and_history_expose_typed_actual_counters_only(admin_client):
metadata = {
"mode": "incremental",
"candidate_organizations_count": 7,
"queried_organizations_count": 3,
"found_memberships_count": 2,
"snapshot_records_count": 10,
"snapshot_organizations_count": 8,
"quarantined_organizations_count": 1,
"completed_organizations_count": 2,
"retained_records_count": 3,
"http_errors_count": 1,
"parse_errors_count": 2,
"upstream_registry_date": None,
"private_loader_setting": "must not expose",
}
job = BackgroundJob.objects.create(
task_id="metadata",
task_name="parsers.sro_membership_check.refresh",
status="started",
meta=metadata,
)
payload = admin_client.get(f"/api/v1/jobs/{job.task_id}/").data
assert payload["meta"]["candidate_organizations_count"] == 7
assert payload["meta"]["source"] == "sro_membership_check"
assert payload["meta"]["upstream_registry_date"] is None
preservation_counters = {
"quarantined_organizations_count": 1,
"completed_organizations_count": 2,
"retained_records_count": 3,
}
for key, value in preservation_counters.items():
assert payload["meta"][key] == value
assert "private_loader_setting" not in payload["meta"]
log = ParserLoadLog.objects.create(
source="sro_membership_check", batch_id=1, status="success", records_count=10
)
ParserSourceArtifact.objects.create(
source="sro_membership_check",
load_batch=1,
status="published",
published_count=10,
metadata=metadata,
)
response = admin_client.get(
reverse("api_v1:system:parser-logs-detail", kwargs={"pk": log.pk})
)
assert response.data["organizations_count"] == 8
assert response.data["meta"]["http_errors_count"] == 1
assert response.data["meta"]["parse_errors_count"] == 2
assert "private_loader_setting" not in response.data["meta"]
for key, value in preservation_counters.items():
assert response.data["meta"][key] == value
schema = admin_client.get(reverse("schema-swagger-ui"), {"format": "openapi"}).data
assert (
schema["definitions"]["SnapshotRunMetadata"]["properties"][
"queried_organizations_count"
]["type"]
== "integer"
)
assert (
"organizations_count"
in schema["definitions"]["SnapshotJobResult"]["properties"]
)
for key in preservation_counters:
for name in ("SnapshotRunMetadata", "SnapshotJobResult"):
assert schema["definitions"][name]["properties"][key]["type"] == "integer"