fix: collect enabled sources weekly including Checko
This commit is contained in:
21
docs/weekly-source-refresh.md
Normal file
21
docs/weekly-source-refresh.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# Недельное обновление источников
|
||||
|
||||
Сбор действующих источников запускается в субботу в 00:00 Europe/Moscow.
|
||||
Миграция 0050 переводит дополнительные действующие расписания, включая три
|
||||
раздела ЧЕКО, на этот cron и отключает три дублирующих interval-расписания.
|
||||
Стабильные имена задач с суффиксами daily/monthly сохранены; фактический период
|
||||
нужно читать из crontab. Отключённые источники с незавершённой приёмкой не включаются.
|
||||
Обслуживание очередей, файлов и ежедневная сборка каталога выгрузок сохраняют свой период.
|
||||
|
||||
Защита ЧЕКО теперь разрешает одну попытку на организацию/раздел/неделю
|
||||
(суббота–пятница МСК). Ошибка не освобождает попытку, параллельные запросы
|
||||
защищены уникальным ограничением БД. Историческое имя колонки period_month
|
||||
сохранено, её значение для новых попыток — дата субботы. Старые попытки не удаляются.
|
||||
Недельный сбор может увеличить расход API относительно месячного.
|
||||
|
||||
Canonical OpenAPI 3 доступен по /openapi.json. Legacy /?format=openapi
|
||||
возвращает Swagger 2 и не используется для генерации фронтенда.
|
||||
|
||||
После релиза проверить миграцию 0050, одинаковую ревизию web/worker/beat,
|
||||
cron Europe/Moscow и отсутствие активных дублей; readiness должен вернуть 200.
|
||||
Проверка регистрации расписания не доказывает успешный сбор с внешних источников.
|
||||
@@ -1,34 +1,37 @@
|
||||
"""Quota-safe monthly claims for external organization lookups."""
|
||||
"""Quota-safe weekly claims for external organization lookups."""
|
||||
|
||||
from datetime import date, datetime
|
||||
from datetime import date, datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from apps.parsers.models import CheckoCollectionAttempt
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
def collection_period_month(at: date | datetime | None = None) -> date:
|
||||
def collection_period_week(at: date | datetime | None = None) -> date:
|
||||
if at is None:
|
||||
local_date = timezone.localdate()
|
||||
local_date = timezone.localdate(timezone=ZoneInfo("Europe/Moscow"))
|
||||
elif isinstance(at, datetime):
|
||||
local_date = (
|
||||
timezone.localtime(at).date() if timezone.is_aware(at) else at.date()
|
||||
timezone.localtime(at, ZoneInfo("Europe/Moscow")).date()
|
||||
if timezone.is_aware(at)
|
||||
else at.date()
|
||||
)
|
||||
else:
|
||||
local_date = at
|
||||
return local_date.replace(day=1)
|
||||
return local_date - timedelta(days=(local_date.weekday() - 5) % 7)
|
||||
|
||||
|
||||
def claim_monthly_collection(
|
||||
def claim_weekly_collection(
|
||||
*,
|
||||
organization_id: str,
|
||||
source: str,
|
||||
at: date | datetime | None = None,
|
||||
) -> CheckoCollectionAttempt | None:
|
||||
"""Atomically claim this month's only allowed external collection attempt."""
|
||||
"""Atomically claim this week's only allowed external collection attempt."""
|
||||
attempt, created = CheckoCollectionAttempt.objects.get_or_create(
|
||||
organization_id=organization_id,
|
||||
source=source,
|
||||
period_month=collection_period_month(at),
|
||||
period_month=collection_period_week(at),
|
||||
defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
|
||||
)
|
||||
return attempt if created else None
|
||||
@@ -40,7 +43,7 @@ def finish_collection(
|
||||
records_count: int,
|
||||
error: Exception | str | None = None,
|
||||
) -> None:
|
||||
"""Persist the outcome without releasing the monthly claim."""
|
||||
"""Persist the outcome without releasing the weekly claim."""
|
||||
attempt.records_count = max(int(records_count), 0)
|
||||
attempt.error_message = str(error or "")
|
||||
attempt.status = (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Monthly, canonical and scoped Checko collection; incomplete organizations survive."""
|
||||
"""Weekly, canonical and scoped Checko collection; incomplete organizations survive."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import re
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
||||
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
|
||||
from apps.parsers.clients.checko import CheckoClient
|
||||
from apps.parsers.fssp_csv import valid_legal_inn
|
||||
@@ -143,7 +143,7 @@ def _collect_scope(client, organizations, source, artifact, raw_file, on_progres
|
||||
counts["organizations_failed"] += 1
|
||||
preserved.append(org.uid)
|
||||
elif (
|
||||
attempt := claim_monthly_collection(
|
||||
attempt := claim_weekly_collection(
|
||||
organization_id=str(org.uid), source=source
|
||||
)
|
||||
) is None:
|
||||
|
||||
61
src/apps/parsers/migrations/0050_weekly_source_refresh.py
Normal file
61
src/apps/parsers/migrations/0050_weekly_source_refresh.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Collect enabled sources once a week on Saturday, Moscow time."""
|
||||
|
||||
from django.db import migrations, models
|
||||
from django.utils import timezone
|
||||
|
||||
# Keep stable schedule IDs/names for existing clients and operational references.
|
||||
EXTRA_SCHEDULES = (
|
||||
"parser:fssp-enforcement-proceedings:daily",
|
||||
"parser:stock-exchange-activity:daily",
|
||||
"parser:checko-leasing:monthly",
|
||||
"parser:checko-messages:monthly",
|
||||
"parser:checko-licenses:monthly",
|
||||
"parser:gosedo-address-directory:daily-msk",
|
||||
"parser:registry-enrichment:daily-msk",
|
||||
)
|
||||
DUPLICATES = (
|
||||
"parse-industrial-production-daily",
|
||||
"parse-manufactures-daily",
|
||||
"parse-industrial-products-daily",
|
||||
)
|
||||
|
||||
|
||||
def weekly_sources(apps, schema_editor):
|
||||
Task = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
|
||||
cron, _ = Cron.objects.get_or_create(
|
||||
minute="0",
|
||||
hour="0",
|
||||
day_of_week="6",
|
||||
day_of_month="*",
|
||||
month_of_year="*",
|
||||
timezone="Europe/Moscow",
|
||||
)
|
||||
for task in Task.objects.filter(enabled=True):
|
||||
if task.name in DUPLICATES:
|
||||
task.enabled = False
|
||||
task.save()
|
||||
elif task.name in EXTRA_SCHEDULES or task.name.endswith(":weekly-saturday-msk"):
|
||||
task.crontab = cron
|
||||
task.interval = task.solar = task.clocked = None
|
||||
task.description = "Weekly source collection: Saturday 00:00 Europe/Moscow."
|
||||
task.save()
|
||||
apps.get_model("django_celery_beat", "PeriodicTasks").objects.update_or_create(
|
||||
ident=1,
|
||||
defaults={"last_update": timezone.now()},
|
||||
)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("parsers", "0049_aggregate_refresh_publication")]
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="checkocollectionattempt",
|
||||
name="period_month",
|
||||
field=models.DateField(
|
||||
"период сбора",
|
||||
help_text="Начало недели сбора (суббота МСК); старые месячные попытки сохранены",
|
||||
),
|
||||
),
|
||||
migrations.RunPython(weekly_sources, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -328,8 +328,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
||||
choices=Source.choices,
|
||||
)
|
||||
period_month = models.DateField(
|
||||
_("месяц сбора"),
|
||||
help_text=_("Первый день календарного месяца, за который занята попытка"),
|
||||
_("период сбора"),
|
||||
help_text=_(
|
||||
"Начало недели сбора (суббота МСК); старые месячные попытки сохранены"
|
||||
),
|
||||
)
|
||||
status = models.CharField(
|
||||
_("статус"),
|
||||
@@ -356,7 +358,7 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})"
|
||||
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m-%d})"
|
||||
|
||||
|
||||
GENERIC_RECORD_SOURCE_CHOICES = [
|
||||
|
||||
@@ -76,7 +76,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
data_scope="Лицензии организаций канонических организаций",
|
||||
task_name="parsers.checko_licenses.refresh",
|
||||
mode="organization_api",
|
||||
access_method="monthly_claim_api",
|
||||
access_method="weekly_claim_api",
|
||||
parser_strategy="scoped_organization_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
@@ -88,7 +88,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
data_scope="Сообщения организаций канонических организаций",
|
||||
task_name="parsers.checko_messages.refresh",
|
||||
mode="organization_api",
|
||||
access_method="monthly_claim_api",
|
||||
access_method="weekly_claim_api",
|
||||
parser_strategy="scoped_organization_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
@@ -100,7 +100,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
data_scope="Лизинг канонических организаций",
|
||||
task_name="parsers.checko_leasing.refresh",
|
||||
mode="organization_api",
|
||||
access_method="monthly_claim_api",
|
||||
access_method="weekly_claim_api",
|
||||
parser_strategy="scoped_organization_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
|
||||
@@ -21,8 +21,8 @@ from apps.core.services import BackgroundJobService
|
||||
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
||||
from apps.parsers.aggregate_publication import claim_part, finish_part
|
||||
from apps.parsers.checko_collection import (
|
||||
claim_monthly_collection,
|
||||
collection_period_month,
|
||||
claim_weekly_collection,
|
||||
collection_period_week,
|
||||
finish_collection,
|
||||
)
|
||||
from apps.parsers.clients.base import HTTPClientError
|
||||
@@ -209,7 +209,7 @@ def _active_registry_lookup_targets(
|
||||
if checko_source is not None:
|
||||
queryset = queryset.exclude(
|
||||
checko_collection_attempts__source=checko_source,
|
||||
checko_collection_attempts__period_month=collection_period_month(),
|
||||
checko_collection_attempts__period_month=collection_period_week(),
|
||||
)
|
||||
queryset = (
|
||||
queryset.order_by("inn", "ogrn", "uid")
|
||||
@@ -887,7 +887,7 @@ def _fetch_checko_bankruptcy_records(
|
||||
)
|
||||
if not targets:
|
||||
logger.info(
|
||||
"No active registry organizations are due for monthly Fedresurs fallback"
|
||||
"No active registry organizations are due for weekly Fedresurs fallback"
|
||||
)
|
||||
return []
|
||||
|
||||
@@ -897,7 +897,7 @@ def _fetch_checko_bankruptcy_records(
|
||||
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
|
||||
records: list[GenericParserItem] = []
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.BANKRUPTCY,
|
||||
)
|
||||
@@ -1117,7 +1117,7 @@ def _fetch_checko_arbitration_records(
|
||||
subjects = _arbitration_subjects(resolved_limit)
|
||||
if not subjects:
|
||||
raise ParserSourceSkipped(
|
||||
"no active registry organizations are due for monthly arbitration lookup"
|
||||
"no active registry organizations are due for weekly arbitration lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1128,7 +1128,7 @@ def _fetch_checko_arbitration_records(
|
||||
failed_lookups = 0
|
||||
attempted_lookups = 0
|
||||
for subject in subjects:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=subject.registry_organization_id,
|
||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||
)
|
||||
@@ -1384,7 +1384,7 @@ def _fetch_checko_registry_inspections(
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no active registry organizations are due for monthly inspections lookup"
|
||||
"no active registry organizations are due for weekly inspections lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1395,7 +1395,7 @@ def _fetch_checko_registry_inspections(
|
||||
failed_lookups = 0
|
||||
attempted_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||
)
|
||||
@@ -1444,7 +1444,7 @@ def _fetch_eis_registry_procurement_records(
|
||||
proxies: list[str] | None,
|
||||
organization_ids: list[str] | None = None,
|
||||
) -> list[GenericParserItem]:
|
||||
"""Fetch addressed EIS notices for the next monthly OПК registry batch."""
|
||||
"""Fetch addressed EIS notices for the next weekly OПК registry batch."""
|
||||
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
||||
if resolved_limit <= 0:
|
||||
return []
|
||||
@@ -1461,7 +1461,7 @@ def _fetch_eis_registry_procurement_records(
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
f"no OПК registry organizations are due for monthly {law}-FZ lookup"
|
||||
f"no OПК registry organizations are due for weekly {law}-FZ lookup"
|
||||
)
|
||||
|
||||
client = EisRegistryProcurementClient(
|
||||
@@ -1472,7 +1472,7 @@ def _fetch_eis_registry_procurement_records(
|
||||
records: list[GenericParserItem] = []
|
||||
failed_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=claim_source,
|
||||
)
|
||||
@@ -1604,7 +1604,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no OПК registry organizations are due for monthly RNP lookup"
|
||||
"no OПК registry organizations are due for weekly RNP lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1615,7 +1615,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
|
||||
failed_lookups = 0
|
||||
rate_limited = False
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
|
||||
)
|
||||
@@ -1780,7 +1780,7 @@ def _fetch_checko_registry_contract_records(
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no active registry organizations are due for monthly contracts lookup"
|
||||
"no active registry organizations are due for weekly contracts lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1791,7 +1791,7 @@ def _fetch_checko_registry_contract_records(
|
||||
failed_lookups = 0
|
||||
attempted_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
from apps.core.models import BackgroundJob
|
||||
from apps.parsers import checko_section_snapshots as snapshots
|
||||
from apps.parsers.checko_collection import claim_monthly_collection
|
||||
from apps.parsers.checko_collection import claim_weekly_collection
|
||||
from apps.parsers.models import ParserSourceArtifact
|
||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||
from apps.parsers.source_cards import SourceCardService
|
||||
@@ -174,7 +174,7 @@ class CheckoSectionContractTests(TestCase):
|
||||
|
||||
def test_no_completed_failure_exposes_rejected_counters(self):
|
||||
source = "checko_messages"
|
||||
claim_monthly_collection(organization_id=str(self.org.uid), source=source)
|
||||
claim_weekly_collection(organization_id=str(self.org.uid), source=source)
|
||||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||||
self.run_task(source)
|
||||
job = BackgroundJob.objects.get()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from datetime import date
|
||||
|
||||
from apps.parsers import tasks as parser_tasks
|
||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
||||
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||
from apps.parsers.models import CheckoCollectionAttempt
|
||||
from django.test import TestCase
|
||||
|
||||
@@ -18,24 +18,24 @@ class CheckoCollectionClaimTest(TestCase):
|
||||
mn_okpo="12345678",
|
||||
)
|
||||
|
||||
def test_claim_is_unique_per_organization_source_and_calendar_month(self):
|
||||
first = claim_monthly_collection(
|
||||
def test_claim_is_unique_per_organization_source_and_calendar_week(self):
|
||||
first = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||
at=date(2026, 7, 1),
|
||||
at=date(2026, 7, 25),
|
||||
)
|
||||
|
||||
duplicate = claim_monthly_collection(
|
||||
duplicate = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||
at=date(2026, 7, 31),
|
||||
)
|
||||
other_source = claim_monthly_collection(
|
||||
other_source = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||
at=date(2026, 7, 31),
|
||||
)
|
||||
next_month = claim_monthly_collection(
|
||||
next_month = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||
at=date(2026, 8, 1),
|
||||
@@ -46,8 +46,8 @@ class CheckoCollectionClaimTest(TestCase):
|
||||
self.assertIsNotNone(other_source)
|
||||
self.assertIsNotNone(next_month)
|
||||
|
||||
def test_failed_attempt_keeps_monthly_claim(self):
|
||||
attempt = claim_monthly_collection(
|
||||
def test_failed_attempt_keeps_weekly_claim(self):
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||
at=date(2026, 7, 19),
|
||||
@@ -55,7 +55,7 @@ class CheckoCollectionClaimTest(TestCase):
|
||||
assert attempt is not None
|
||||
finish_collection(attempt, records_count=0, error="quota exhausted")
|
||||
|
||||
repeated = claim_monthly_collection(
|
||||
repeated = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||
at=date(2026, 7, 20),
|
||||
@@ -78,7 +78,7 @@ class CheckoCollectionClaimTest(TestCase):
|
||||
)
|
||||
second.opk_registry_membership = True
|
||||
second.save(update_fields=["opk_registry_membership"])
|
||||
claimed = claim_monthly_collection(
|
||||
claimed = claim_weekly_collection(
|
||||
organization_id=str(self.organization.id),
|
||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||
)
|
||||
@@ -115,3 +115,15 @@ class CheckoCollectionClaimTest(TestCase):
|
||||
[target.organization_id for target in targets],
|
||||
[str(self.organization.id)],
|
||||
)
|
||||
|
||||
|
||||
def test_week_boundary_uses_moscow_saturday():
|
||||
from datetime import datetime, timezone
|
||||
from apps.parsers.checko_collection import collection_period_week
|
||||
|
||||
assert collection_period_week(
|
||||
datetime(2026, 10, 30, 20, 59, tzinfo=timezone.utc)
|
||||
) == date(2026, 10, 24)
|
||||
assert collection_period_week(
|
||||
datetime(2026, 10, 30, 21, 0, tzinfo=timezone.utc)
|
||||
) == date(2026, 10, 31)
|
||||
|
||||
@@ -6,7 +6,7 @@ from tempfile import TemporaryDirectory
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
||||
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||
from apps.parsers.models import (
|
||||
CheckoCollectionAttempt,
|
||||
ParserSourceArtifact,
|
||||
@@ -96,7 +96,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
def test_claimed_organization_keeps_previous_records(self):
|
||||
self.seed(self.orgs[0])
|
||||
before = self.existing(self.orgs[0])
|
||||
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
|
||||
claim_weekly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
|
||||
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
|
||||
assert self.existing(self.orgs[0]) == before
|
||||
assert collector.call_count == 1
|
||||
@@ -134,7 +134,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
for org in self.orgs:
|
||||
self.seed(org)
|
||||
before = self.existing(self.orgs[1])
|
||||
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
|
||||
claim_weekly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
|
||||
self.run_refresh(rows=[[]])
|
||||
assert not self.existing(self.orgs[0])
|
||||
assert self.existing(self.orgs[1]) == before
|
||||
@@ -163,7 +163,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
)
|
||||
assert attempt.status == attempt.Status.FAILED
|
||||
assert (
|
||||
claim_monthly_collection(
|
||||
claim_weekly_collection(
|
||||
organization_id=str(self.orgs[0].uid), source=SOURCE
|
||||
)
|
||||
is None
|
||||
@@ -172,7 +172,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
def test_no_completed_scope_rejects_without_publication(self):
|
||||
for org in self.orgs:
|
||||
self.seed(org)
|
||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
before = [self.existing(org) for org in self.orgs]
|
||||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||||
self.run_refresh()
|
||||
@@ -220,7 +220,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
|
||||
assert attempt.status == attempt.Status.FAILED
|
||||
assert (
|
||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
is None
|
||||
)
|
||||
|
||||
@@ -246,7 +246,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
before = [self.existing(org)[0][0] for org in self.orgs]
|
||||
# Advance the calendar; never delete or reset a monthly claim.
|
||||
with patch(
|
||||
"apps.parsers.checko_collection.collection_period_month",
|
||||
"apps.parsers.checko_collection.collection_period_week",
|
||||
return_value=date(2026, 11, 1),
|
||||
):
|
||||
self.run_refresh(load_batch=2)
|
||||
@@ -257,13 +257,13 @@ class CheckoSectionSnapshotTests(TestCase):
|
||||
CheckoCollectionAttempt.Source.values
|
||||
)
|
||||
for source in adapter().CHECKO_SECTIONS:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=str(self.orgs[0].uid), source=source
|
||||
)
|
||||
assert attempt is not None
|
||||
finish_collection(attempt, records_count=0, error="fixture")
|
||||
assert (
|
||||
claim_monthly_collection(
|
||||
claim_weekly_collection(
|
||||
organization_id=str(self.orgs[0].uid), source=source
|
||||
)
|
||||
is None
|
||||
@@ -311,7 +311,7 @@ def test_concurrent_monthly_claim_one_winner():
|
||||
|
||||
def claim():
|
||||
return (
|
||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
@@ -161,3 +161,41 @@ class RequestedSourceScheduleTest(TestCase):
|
||||
migration.disable_schedules(apps, None)
|
||||
assert not PeriodicTask.objects.filter(name__in=expected, enabled=True).exists()
|
||||
assert PeriodicTask.objects.filter(name__in=expected).count() == 5
|
||||
|
||||
|
||||
class WeeklySourcePolicyTest(TestCase):
|
||||
def test_policy_consolidates_enabled_sources_and_preserves_disabled_gates(self):
|
||||
migration = importlib.import_module(
|
||||
"apps.parsers.migrations.0050_weekly_source_refresh"
|
||||
)
|
||||
interval = IntervalSchedule.objects.create(every=30, period="days")
|
||||
source, _ = PeriodicTask.objects.update_or_create(
|
||||
name="parser:checko-leasing:monthly",
|
||||
defaults={
|
||||
"task": "parsers.checko_leasing.refresh",
|
||||
"enabled": True,
|
||||
"interval": interval,
|
||||
"crontab": None,
|
||||
},
|
||||
)
|
||||
gated, _ = PeriodicTask.objects.update_or_create(
|
||||
name="parser:sro-membership:full-msk",
|
||||
defaults={
|
||||
"task": "parsers.sro_membership_check.refresh",
|
||||
"enabled": False,
|
||||
"interval": interval,
|
||||
},
|
||||
)
|
||||
migration.weekly_sources(apps, None)
|
||||
migration.weekly_sources(apps, None)
|
||||
source.refresh_from_db()
|
||||
gated.refresh_from_db()
|
||||
self.assertIsNone(source.interval_id)
|
||||
self.assertEqual(source.crontab.day_of_week, "6")
|
||||
self.assertEqual(str(source.crontab.timezone), "Europe/Moscow")
|
||||
self.assertFalse(gated.enabled)
|
||||
self.assertFalse(
|
||||
PeriodicTask.objects.filter(
|
||||
name__in=migration.DUPLICATES, enabled=True
|
||||
).exists()
|
||||
)
|
||||
|
||||
@@ -324,7 +324,7 @@ class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase):
|
||||
patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient),
|
||||
patch.object(
|
||||
parser_tasks,
|
||||
"claim_monthly_collection",
|
||||
"claim_weekly_collection",
|
||||
return_value=SimpleNamespace(),
|
||||
),
|
||||
patch.object(parser_tasks, "finish_collection"),
|
||||
|
||||
Reference in New Issue
Block a user