Files
mostovik-backend/tests/apps/parsers/test_refresh_progress.py
Aleksandr Meshchryakov 9b3da5b505
Some checks failed
Mostovik Backend CI/CD / Tests and lint (push) Successful in 3m16s
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
fix: implement backend testing amendments and atomic source publication
2026-10-12 09:47:28 +02:00

207 lines
7.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Общий прогресс ручного запуска, в том числе после завершения его частей."""
from datetime import timedelta
from unittest.mock import MagicMock
from uuid import UUID
import pytest
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.source_cards import SourceCardService
from django.test import override_settings
from django.utils import timezone
@pytest.mark.django_db
def test_all_jobs_are_visible_before_dispatch_and_completed_parts_stay_in_progress():
definition = SourceCardService.get_definition("manufacturers-and-products")
observed_counts = []
tasks = [MagicMock(), MagicMock(), MagicMock()]
def dispatch(**kwargs):
jobs = list(BackgroundJob.objects.all())
observed_counts.append(len(jobs))
assert {job.task_id for job in jobs} == set(jobs[0].meta["refresh_task_ids"])
for task in tasks:
task.apply_async.side_effect = dispatch
specs = tuple(
(task, name, source)
for task, name, source in zip(
tasks,
[
"apps.parsers.tasks.parse_industrial_production",
"apps.parsers.tasks.parse_industrial_products",
"apps.parsers.tasks.parse_manufactures",
],
["industrial", "industrial_products", "manufactures"],
strict=False,
)
)
result = SourceCardService._enqueue_refresh_group(
definition, specs, requested_by_id=1, kwargs={}
)
assert observed_counts == [3, 3, 3]
jobs = [BackgroundJob.objects.get(task_id=item["task_id"]) for item in result]
jobs[0].complete()
jobs[1].update_progress(50)
jobs[2].update_progress(0)
card = SourceCardService.get_card(definition.slug)
assert card["progress"] == 50
assert len(card["active_tasks"]) == 2
jobs[1].complete()
assert SourceCardService.get_card(definition.slug)["progress"] == 67
jobs[2].complete()
for card in [
SourceCardService.get_card(definition.slug),
next(
card
for card in SourceCardService.list_cards()
if card["slug"] == definition.slug
),
]:
assert (card["progress"], card["status"], card["active_tasks"]) == (
100,
"success",
[],
)
@pytest.mark.django_db
def test_failed_run_keeps_progress_and_error_after_last_active_job_disappears():
job = BackgroundJob.objects.create(
task_id="failed-refresh",
task_name="apps.parsers.tasks.parse_arbitration_cases",
meta={
"source_card": "arbitration-cases",
"refresh_task_ids": ["failed-refresh"],
},
)
job.update_progress(60)
job.fail("Ошибка загрузки")
card = SourceCardService.get_card("arbitration-cases")
assert (card["status"], card["progress"], card["error_message"]) == (
"failed",
60,
"Не удалось обновить данные. Повторите попытку позже.",
)
@pytest.mark.django_db
def test_broker_failure_keeps_all_registered_parts_in_final_failed_state():
definition = SourceCardService.get_definition("defense-unreliable-suppliers")
task = MagicMock()
task.apply_async.side_effect = RuntimeError("broker unavailable")
with pytest.raises(RuntimeError):
SourceCardService._enqueue_refresh_group(
definition,
[
(task, "test.first", "unfair_suppliers"),
(task, "test.second", "fas_goz"),
],
requested_by_id=1,
kwargs={},
)
jobs = list(BackgroundJob.objects.all())
assert len(jobs) == 2
assert all(job.status == JobStatus.FAILURE and job.error for job in jobs)
assert task.apply_async.call_count == 1
@pytest.mark.django_db
def test_completed_new_run_does_not_hide_active_older_run():
definition = SourceCardService.get_definition("manufacturers-and-products")
# Historical overlapping runs predate the publication barrier. New refreshes
# reject this overlap, but polling must still represent existing older jobs.
def legacy_run(prefix):
ids = [f"{prefix}-{index}" for index in range(len(definition.task_names))]
for task_id, name in zip(ids, definition.task_names, strict=True):
BackgroundJob.objects.create(
task_id=task_id,
task_name=name,
meta={"source_card": definition.slug, "refresh_task_ids": ids},
)
return [{"task_id": task_id} for task_id in ids]
first = legacy_run("older")
second = legacy_run("newer")
for item in second:
BackgroundJob.objects.get(task_id=item["task_id"]).complete()
card = SourceCardService.get_card(definition.slug)
assert {job["task_id"] for job in card["active_tasks"]} == {
job["task_id"] for job in first
}
assert (card["status"], card["progress"]) == ("in_progress", 50)
for item in first:
BackgroundJob.objects.get(task_id=item["task_id"]).complete()
assert SourceCardService.get_card(definition.slug)["progress"] == 100
@pytest.mark.django_db
@pytest.mark.parametrize(
"status,time_field",
[(JobStatus.PENDING, "created_at"), (JobStatus.STARTED, "updated_at")],
)
@override_settings(
PARSER_STALE_LOAD_MAX_AGE_MINUTES=10, PARSER_STALE_PENDING_JOB_MAX_AGE_MINUTES=10
)
def test_latest_task_does_not_reactivate_a_stale_job(status, time_field):
definition = SourceCardService.get_definition("arbitration-cases")
job = BackgroundJob.objects.create(
task_id="stale-task", task_name=definition.task_names[0], status=status
)
BackgroundJob.objects.filter(pk=job.pk).update(
**{time_field: timezone.now() - timedelta(minutes=15)}
)
assert SourceCardService.get_card(definition.slug)["active_tasks"] == []
@pytest.mark.django_db
def test_latest_job_uses_created_time_then_uuid_and_keeps_older_active_jobs():
definition = SourceCardService.get_definition("arbitration-cases")
jobs = [
BackgroundJob.objects.create(
id=UUID(int=index),
task_id=f"ordered-{index}",
task_name=definition.task_names[0],
status=JobStatus.SUCCESS,
)
for index in (1, 2, 3, 4)
]
now = timezone.now()
BackgroundJob.objects.filter(pk__in=[job.pk for job in jobs]).update(
created_at=now - timedelta(minutes=2)
)
# UUID 4 is older; UUID 2 wins the timestamp tie with UUID 1.
BackgroundJob.objects.filter(pk__in=[jobs[0].pk, jobs[1].pk]).update(created_at=now)
BackgroundJob.objects.filter(pk=jobs[2].pk).update(status=JobStatus.STARTED)
active, refresh = SourceCardService._build_task_maps((definition,))
assert [job["task_id"] for job in active[definition.slug]] == ["ordered-3"]
assert [job["task_id"] for job in refresh[definition.slug]] == [
"ordered-2",
"ordered-3",
]
@pytest.mark.django_db
def test_latest_job_is_chosen_before_filtering_source_card_metadata():
definition = SourceCardService.get_definition("arbitration-cases")
older = BackgroundJob.objects.create(
task_id="old-card-job",
task_name="custom.shared.task",
status=JobStatus.SUCCESS,
meta={"source_card": definition.slug},
)
newer = BackgroundJob.objects.create(
task_id="new-other-card-job",
task_name=older.task_name,
status=JobStatus.SUCCESS,
meta={"source_card": "bankruptcy-procedures"},
)
BackgroundJob.objects.filter(pk=older.pk).update(
created_at=newer.created_at - timedelta(minutes=1)
)
active, refresh = SourceCardService._build_task_maps((definition,))
assert active == {}
assert refresh == {}