Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m16s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
230 lines
9.9 KiB
Python
230 lines
9.9 KiB
Python
"""Three independent source APIs and full/partial task semantics."""
|
||
|
||
from importlib import import_module
|
||
from tempfile import TemporaryDirectory
|
||
from unittest.mock import patch
|
||
|
||
import pytest
|
||
from apps.core.models import BackgroundJob
|
||
from apps.parsers import checko_section_snapshots as snapshots
|
||
from apps.parsers.checko_collection import claim_monthly_collection
|
||
from apps.parsers.models import ParserSourceArtifact
|
||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||
from apps.parsers.source_cards import SourceCardService
|
||
from core.celery import app as celery_app
|
||
from django.core.cache import cache
|
||
from django.test import TestCase, override_settings
|
||
from django.utils import timezone
|
||
from organizations.models import Organization
|
||
from organizations.source_groups import get_source_group_descriptor
|
||
from organizations.source_record_export import _build_record_row, _source_group_queryset
|
||
from rest_framework.test import APIClient
|
||
|
||
from tests.apps.parsers.test_checko_sections import client_with, fixture
|
||
from tests.apps.user.factories import UserFactory
|
||
|
||
SOURCES = {
|
||
"checko_leasing": ("leasing", "leasing_contract"),
|
||
"checko_messages": ("company_messages", "company_message"),
|
||
"checko_licenses": ("company_licenses", "company_license"),
|
||
}
|
||
|
||
|
||
class CheckoSectionContractTests(TestCase):
|
||
def setUp(self):
|
||
tmp = TemporaryDirectory()
|
||
self.addCleanup(tmp.cleanup)
|
||
settings = override_settings(
|
||
MEDIA_ROOT=tmp.name + "/media",
|
||
PARSER_PRIVATE_ARTIFACT_ROOT=tmp.name + "/private",
|
||
)
|
||
settings.enable()
|
||
self.addCleanup(settings.disable)
|
||
cache.clear()
|
||
self.org = Organization.objects.create(
|
||
name="АО Проверка",
|
||
inn="7707083893",
|
||
ogrn="1027700132195",
|
||
okpo="12345678",
|
||
directory_imported_at=timezone.now(),
|
||
gk_code="rosatom",
|
||
)
|
||
self.client = APIClient()
|
||
self.client.force_authenticate(UserFactory.create_user())
|
||
self.url = "/api/v2/organization-source-records/"
|
||
|
||
def transport(self, source):
|
||
return (
|
||
client_with(fixture("license.json"))
|
||
if source == "checko_licenses"
|
||
else client_with(
|
||
fixture("messages-page-1.json"), fixture("messages-page-2.json")
|
||
)
|
||
)
|
||
|
||
def run_task(self, source):
|
||
import_module("apps.parsers.tasks")
|
||
task = celery_app.tasks[f"parsers.{source}.refresh"]
|
||
with patch.object(snapshots, "_client", return_value=self.transport(source)):
|
||
return task.apply(throw=True).result
|
||
|
||
def test_three_independent_groups_and_typed_list_detail_exports(self):
|
||
for source, (group, kind) in SOURCES.items():
|
||
descriptor = get_source_group_descriptor(source)
|
||
assert (descriptor.source_group, descriptor.record_type) == (group, kind)
|
||
with patch.object(
|
||
snapshots, "_client", return_value=self.transport(source)
|
||
):
|
||
snapshots.refresh_checko_section(source=source, load_batch=1)
|
||
response = self.client.get(self.url, {"source_group": group})
|
||
assert response.status_code == 200, response.data
|
||
assert response.data["data"], response.data
|
||
for row in response.data["data"]:
|
||
assert row["source"] == source
|
||
assert row["record_type"] == kind
|
||
assert row["amount"] is None
|
||
assert row["payload"]["fetched_at"].endswith("Z")
|
||
assert row["organization"]["inn"] == self.org.inn
|
||
detail = self.client.get(self.url + row["uid"] + "/")
|
||
assert detail.status_code == 200, detail.data
|
||
assert detail.data["payload"] == row["payload"]
|
||
record = _source_group_queryset(group, export_year=2026).get(
|
||
uid=row["uid"]
|
||
)
|
||
exported = _build_record_row(record, include_financial_lines=False)
|
||
assert exported["source"] == source
|
||
assert exported["payload.source_url"] == row["payload"]["source_url"]
|
||
if source == "checko_licenses":
|
||
assert response.data["data"][0]["payload"]["valid_until"] is None
|
||
|
||
def test_partial_does_not_move_full_last_success_and_keeps_mixed_batches_visible(
|
||
self,
|
||
):
|
||
source = "checko_messages"
|
||
full = self.run_task(source)
|
||
assert full["completeness"] == "full"
|
||
before = SourceCardService.get_card("checko-messages")["latest_success_load"][
|
||
"batch_id"
|
||
]
|
||
org2 = Organization.objects.create(
|
||
name="АО Второй",
|
||
inn="7702070139",
|
||
ogrn="1027700070518",
|
||
okpo="87654321",
|
||
directory_imported_at=timezone.now(),
|
||
opk_registry_membership=True,
|
||
)
|
||
original_collect = snapshots.collect_checko_section
|
||
|
||
def collected(client, *, organization, source):
|
||
# Real adapter, identical structure for the second canonical identity.
|
||
responses = [
|
||
fixture("messages-page-1.json"),
|
||
fixture("messages-page-2.json"),
|
||
]
|
||
for page in responses:
|
||
page["company"] = {"ИНН": org2.inn, "ОГРН": org2.ogrn}
|
||
for row in page["data"]["Записи"]:
|
||
row["Публикатор"] = org2.ogrn
|
||
row["Участники"] = [org2.inn]
|
||
yield from original_collect(
|
||
client_with(*responses), organization=organization, source=source
|
||
)
|
||
|
||
with patch.object(snapshots, "collect_checko_section", collected):
|
||
partial = self.run_task(source)
|
||
assert partial["completeness"] == "partial"
|
||
assert partial["quota_skipped"] == partial["organizations_preserved"] == 1
|
||
assert partial["organizations_updated"] == 1
|
||
card = SourceCardService.get_card("checko-messages")
|
||
assert card["latest_success_load"]["batch_id"] == before
|
||
assert card["latest_load"]["meta"]["completeness"] == "partial"
|
||
response = self.client.get(self.url, {"source_group": "company_messages"})
|
||
assert response.status_code == 200, response.data
|
||
assert {row["load_batch"] for row in response.data["data"]} == {
|
||
full["batch_id"],
|
||
partial["batch_id"],
|
||
}
|
||
assert len(response.data["data"]) == 4
|
||
assert (
|
||
BackgroundJob.objects.order_by("-created_at").first().result["completeness"]
|
||
== "partial"
|
||
)
|
||
|
||
def test_nullable_publisher_survives_public_list_and_detail(self):
|
||
first = fixture("messages-page-1.json")
|
||
first["data"]["Записи"][0]["Публикатор"] = None
|
||
with patch.object(
|
||
snapshots,
|
||
"_client",
|
||
return_value=client_with(first, fixture("messages-page-2.json")),
|
||
):
|
||
snapshots.refresh_checko_section(source="checko_messages", load_batch=1)
|
||
response = self.client.get(self.url, {"source_group": "company_messages"})
|
||
assert response.status_code == 200, response.data
|
||
row = next(
|
||
row
|
||
for row in response.data["data"]
|
||
if row["payload"]["publisher_identifier"] is None
|
||
)
|
||
detail = self.client.get(self.url + row["uid"] + "/")
|
||
assert detail.status_code == 200, detail.data
|
||
assert detail.data["payload"] == row["payload"]
|
||
assert row["payload"]["participant_identifiers"] == [self.org.inn]
|
||
|
||
def test_no_completed_failure_exposes_rejected_counters(self):
|
||
source = "checko_messages"
|
||
claim_monthly_collection(organization_id=str(self.org.uid), source=source)
|
||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||
self.run_task(source)
|
||
job = BackgroundJob.objects.get()
|
||
assert job.status == "failed"
|
||
assert job.result["quota_skipped"] == 1
|
||
assert job.result["organizations_preserved"] == 1
|
||
assert job.result["completeness"] == "partial"
|
||
assert job.result["artifact_id"] == str(ParserSourceArtifact.objects.get().uid)
|
||
assert (
|
||
SourceCardService.get_card("checko-messages")["latest_success_load"] is None
|
||
)
|
||
|
||
|
||
@pytest.mark.django_db
|
||
@pytest.mark.parametrize("source", SOURCES)
|
||
def test_refresh_and_run_share_guard_permissions_and_no_file_url(source):
|
||
import_module("apps.parsers.tasks")
|
||
task = celery_app.tasks[f"parsers.{source}.refresh"]
|
||
client = APIClient()
|
||
client.force_authenticate(UserFactory.create_user())
|
||
slug = source.replace("_", "-")
|
||
urls = [f"/api/v1/sources/{slug}/refresh/", f"/api/v1/parsers/run/{source}/"]
|
||
for url in urls:
|
||
assert client.post(url, {}, format="json").status_code == 403
|
||
client.force_authenticate(UserFactory.create_user(is_staff=True))
|
||
for url in urls:
|
||
assert (
|
||
client.post(
|
||
url, {"params": {"file_url": "https://foreign.invalid"}}, format="json"
|
||
).status_code
|
||
== 400
|
||
)
|
||
with patch.object(task, "apply_async"):
|
||
assert client.post(urls[0], {}, format="json").status_code == 202
|
||
assert client.post(urls[1], {}, format="json").status_code == 409
|
||
assert BackgroundJob.objects.count() == 1
|
||
|
||
|
||
def test_checko_discriminated_variants_in_canonical_document():
|
||
import json
|
||
from pathlib import Path
|
||
|
||
document = json.loads(
|
||
(Path(__file__).parents[3] / "src/core/openapi.json").read_text()
|
||
)
|
||
schemas = document["components"]["schemas"]
|
||
for name in ("OrganizationSourceRecordList", "OrganizationSourceRecord"):
|
||
mapping = schemas[name]["discriminator"]["mapping"]
|
||
for source, (group, kind) in SOURCES.items():
|
||
variant = schemas[mapping[group].rsplit("/", 1)[1]]
|
||
assert variant["properties"]["source"]["enum"] == [source]
|
||
assert variant["properties"]["record_type"]["enum"] == [kind]
|