fix: collect enabled sources weekly including Checko

This commit is contained in:
Aleksandr Meshchryakov
2026-10-29 17:46:26 +01:00
parent aeade84761
commit bc9767244d
12 changed files with 196 additions and 59 deletions

View File

@@ -0,0 +1,21 @@
# Недельное обновление источников
Сбор действующих источников запускается в субботу в 00:00 Europe/Moscow.
Миграция 0050 переводит дополнительные действующие расписания, включая три
раздела ЧЕКО, на этот cron и отключает три дублирующих interval-расписания.
Стабильные имена задач с суффиксами daily/monthly сохранены; фактический период
нужно читать из crontab. Отключённые источники с незавершённой приёмкой не включаются.
Обслуживание очередей, файлов и ежедневная сборка каталога выгрузок сохраняют свой период.
Защита ЧЕКО теперь разрешает одну попытку на организацию/раздел/неделю
(суббота–пятница МСК). Ошибка не освобождает попытку, параллельные запросы
защищены уникальным ограничением БД. Историческое имя колонки period_month
сохранено, её значение для новых попыток — дата субботы. Старые попытки не удаляются.
Недельный сбор может увеличить расход API относительно месячного.
Canonical OpenAPI 3 доступен по /openapi.json. Legacy /?format=openapi
возвращает Swagger 2 и не используется для генерации фронтенда.
После релиза проверить миграцию 0050, одинаковую ревизию web/worker/beat,
cron Europe/Moscow и отсутствие активных дублей; readiness должен вернуть 200.
Проверка регистрации расписания не доказывает успешный сбор с внешних источников.

View File

@@ -1,34 +1,37 @@
"""Quota-safe monthly claims for external organization lookups."""
"""Quota-safe weekly claims for external organization lookups."""
from datetime import date, datetime
from datetime import date, datetime, timedelta
from zoneinfo import ZoneInfo
from apps.parsers.models import CheckoCollectionAttempt
from django.utils import timezone
def collection_period_month(at: date | datetime | None = None) -> date:
def collection_period_week(at: date | datetime | None = None) -> date:
if at is None:
local_date = timezone.localdate()
local_date = timezone.localdate(timezone=ZoneInfo("Europe/Moscow"))
elif isinstance(at, datetime):
local_date = (
timezone.localtime(at).date() if timezone.is_aware(at) else at.date()
timezone.localtime(at, ZoneInfo("Europe/Moscow")).date()
if timezone.is_aware(at)
else at.date()
)
else:
local_date = at
return local_date.replace(day=1)
return local_date - timedelta(days=(local_date.weekday() - 5) % 7)
def claim_monthly_collection(
def claim_weekly_collection(
*,
organization_id: str,
source: str,
at: date | datetime | None = None,
) -> CheckoCollectionAttempt | None:
"""Atomically claim this month's only allowed external collection attempt."""
"""Atomically claim this week's only allowed external collection attempt."""
attempt, created = CheckoCollectionAttempt.objects.get_or_create(
organization_id=organization_id,
source=source,
period_month=collection_period_month(at),
period_month=collection_period_week(at),
defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
)
return attempt if created else None
@@ -40,7 +43,7 @@ def finish_collection(
records_count: int,
error: Exception | str | None = None,
) -> None:
"""Persist the outcome without releasing the monthly claim."""
"""Persist the outcome without releasing the weekly claim."""
attempt.records_count = max(int(records_count), 0)
attempt.error_message = str(error or "")
attempt.status = (

View File

@@ -1,4 +1,4 @@
"""Monthly, canonical and scoped Checko collection; incomplete organizations survive."""
"""Weekly, canonical and scoped Checko collection; incomplete organizations survive."""
from __future__ import annotations
@@ -6,7 +6,7 @@ import json
import re
from tempfile import TemporaryFile
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
from apps.parsers.clients.checko import CheckoClient
from apps.parsers.fssp_csv import valid_legal_inn
@@ -143,7 +143,7 @@ def _collect_scope(client, organizations, source, artifact, raw_file, on_progres
counts["organizations_failed"] += 1
preserved.append(org.uid)
elif (
attempt := claim_monthly_collection(
attempt := claim_weekly_collection(
organization_id=str(org.uid), source=source
)
) is None:

View File

@@ -0,0 +1,61 @@
"""Collect enabled sources once a week on Saturday, Moscow time."""
from django.db import migrations, models
from django.utils import timezone
# Keep stable schedule IDs/names for existing clients and operational references.
EXTRA_SCHEDULES = (
"parser:fssp-enforcement-proceedings:daily",
"parser:stock-exchange-activity:daily",
"parser:checko-leasing:monthly",
"parser:checko-messages:monthly",
"parser:checko-licenses:monthly",
"parser:gosedo-address-directory:daily-msk",
"parser:registry-enrichment:daily-msk",
)
DUPLICATES = (
"parse-industrial-production-daily",
"parse-manufactures-daily",
"parse-industrial-products-daily",
)
def weekly_sources(apps, schema_editor):
Task = apps.get_model("django_celery_beat", "PeriodicTask")
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
cron, _ = Cron.objects.get_or_create(
minute="0",
hour="0",
day_of_week="6",
day_of_month="*",
month_of_year="*",
timezone="Europe/Moscow",
)
for task in Task.objects.filter(enabled=True):
if task.name in DUPLICATES:
task.enabled = False
task.save()
elif task.name in EXTRA_SCHEDULES or task.name.endswith(":weekly-saturday-msk"):
task.crontab = cron
task.interval = task.solar = task.clocked = None
task.description = "Weekly source collection: Saturday 00:00 Europe/Moscow."
task.save()
apps.get_model("django_celery_beat", "PeriodicTasks").objects.update_or_create(
ident=1,
defaults={"last_update": timezone.now()},
)
class Migration(migrations.Migration):
dependencies = [("parsers", "0049_aggregate_refresh_publication")]
operations = [
migrations.AlterField(
model_name="checkocollectionattempt",
name="period_month",
field=models.DateField(
"период сбора",
help_text="Начало недели сбора (суббота МСК); старые месячные попытки сохранены",
),
),
migrations.RunPython(weekly_sources, migrations.RunPython.noop),
]

View File

@@ -328,8 +328,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
choices=Source.choices,
)
period_month = models.DateField(
_("месяц сбора"),
help_text=_("Первый день календарного месяца, за который занята попытка"),
_("период сбора"),
help_text=_(
"Начало недели сбора (суббота МСК); старые месячные попытки сохранены"
),
)
status = models.CharField(
_("статус"),
@@ -356,7 +358,7 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
]
def __str__(self) -> str:
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})"
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m-%d})"
GENERIC_RECORD_SOURCE_CHOICES = [

View File

@@ -76,7 +76,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
data_scope="Лицензии организаций канонических организаций",
task_name="parsers.checko_licenses.refresh",
mode="organization_api",
access_method="monthly_claim_api",
access_method="weekly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
@@ -88,7 +88,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
data_scope="Сообщения организаций канонических организаций",
task_name="parsers.checko_messages.refresh",
mode="organization_api",
access_method="monthly_claim_api",
access_method="weekly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
@@ -100,7 +100,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
data_scope="Лизинг канонических организаций",
task_name="parsers.checko_leasing.refresh",
mode="organization_api",
access_method="monthly_claim_api",
access_method="weekly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),

View File

@@ -21,8 +21,8 @@ from apps.core.services import BackgroundJobService
from apps.core.tasks import PeriodicTask as CorePeriodicTask
from apps.parsers.aggregate_publication import claim_part, finish_part
from apps.parsers.checko_collection import (
claim_monthly_collection,
collection_period_month,
claim_weekly_collection,
collection_period_week,
finish_collection,
)
from apps.parsers.clients.base import HTTPClientError
@@ -209,7 +209,7 @@ def _active_registry_lookup_targets(
if checko_source is not None:
queryset = queryset.exclude(
checko_collection_attempts__source=checko_source,
checko_collection_attempts__period_month=collection_period_month(),
checko_collection_attempts__period_month=collection_period_week(),
)
queryset = (
queryset.order_by("inn", "ogrn", "uid")
@@ -887,7 +887,7 @@ def _fetch_checko_bankruptcy_records(
)
if not targets:
logger.info(
"No active registry organizations are due for monthly Fedresurs fallback"
"No active registry organizations are due for weekly Fedresurs fallback"
)
return []
@@ -897,7 +897,7 @@ def _fetch_checko_bankruptcy_records(
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
records: list[GenericParserItem] = []
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.BANKRUPTCY,
)
@@ -1117,7 +1117,7 @@ def _fetch_checko_arbitration_records(
subjects = _arbitration_subjects(resolved_limit)
if not subjects:
raise ParserSourceSkipped(
"no active registry organizations are due for monthly arbitration lookup"
"no active registry organizations are due for weekly arbitration lookup"
)
checko_proxies = (
@@ -1128,7 +1128,7 @@ def _fetch_checko_arbitration_records(
failed_lookups = 0
attempted_lookups = 0
for subject in subjects:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=subject.registry_organization_id,
source=CheckoCollectionAttempt.Source.ARBITRATION,
)
@@ -1384,7 +1384,7 @@ def _fetch_checko_registry_inspections(
)
if not targets:
raise ParserSourceSkipped(
"no active registry organizations are due for monthly inspections lookup"
"no active registry organizations are due for weekly inspections lookup"
)
checko_proxies = (
@@ -1395,7 +1395,7 @@ def _fetch_checko_registry_inspections(
failed_lookups = 0
attempted_lookups = 0
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.INSPECTIONS,
)
@@ -1444,7 +1444,7 @@ def _fetch_eis_registry_procurement_records(
proxies: list[str] | None,
organization_ids: list[str] | None = None,
) -> list[GenericParserItem]:
"""Fetch addressed EIS notices for the next monthly OПК registry batch."""
"""Fetch addressed EIS notices for the next weekly OПК registry batch."""
resolved_limit = _resolve_registry_enrichment_limit(limit)
if resolved_limit <= 0:
return []
@@ -1461,7 +1461,7 @@ def _fetch_eis_registry_procurement_records(
)
if not targets:
raise ParserSourceSkipped(
f"no OПК registry organizations are due for monthly {law}-FZ lookup"
f"no OПК registry organizations are due for weekly {law}-FZ lookup"
)
client = EisRegistryProcurementClient(
@@ -1472,7 +1472,7 @@ def _fetch_eis_registry_procurement_records(
records: list[GenericParserItem] = []
failed_lookups = 0
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=claim_source,
)
@@ -1604,7 +1604,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
)
if not targets:
raise ParserSourceSkipped(
"no OПК registry organizations are due for monthly RNP lookup"
"no OПК registry organizations are due for weekly RNP lookup"
)
checko_proxies = (
@@ -1615,7 +1615,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
failed_lookups = 0
rate_limited = False
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
)
@@ -1780,7 +1780,7 @@ def _fetch_checko_registry_contract_records(
)
if not targets:
raise ParserSourceSkipped(
"no active registry organizations are due for monthly contracts lookup"
"no active registry organizations are due for weekly contracts lookup"
)
checko_proxies = (
@@ -1791,7 +1791,7 @@ def _fetch_checko_registry_contract_records(
failed_lookups = 0
attempted_lookups = 0
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.CONTRACTS,
)

View File

@@ -7,7 +7,7 @@ from unittest.mock import patch
import pytest
from apps.core.models import BackgroundJob
from apps.parsers import checko_section_snapshots as snapshots
from apps.parsers.checko_collection import claim_monthly_collection
from apps.parsers.checko_collection import claim_weekly_collection
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.source_cards import SourceCardService
@@ -174,7 +174,7 @@ class CheckoSectionContractTests(TestCase):
def test_no_completed_failure_exposes_rejected_counters(self):
source = "checko_messages"
claim_monthly_collection(organization_id=str(self.org.uid), source=source)
claim_weekly_collection(organization_id=str(self.org.uid), source=source)
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_task(source)
job = BackgroundJob.objects.get()

View File

@@ -1,7 +1,7 @@
from datetime import date
from apps.parsers import tasks as parser_tasks
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.models import CheckoCollectionAttempt
from django.test import TestCase
@@ -18,24 +18,24 @@ class CheckoCollectionClaimTest(TestCase):
mn_okpo="12345678",
)
def test_claim_is_unique_per_organization_source_and_calendar_month(self):
first = claim_monthly_collection(
def test_claim_is_unique_per_organization_source_and_calendar_week(self):
first = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.ARBITRATION,
at=date(2026, 7, 1),
at=date(2026, 7, 25),
)
duplicate = claim_monthly_collection(
duplicate = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.ARBITRATION,
at=date(2026, 7, 31),
)
other_source = claim_monthly_collection(
other_source = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.INSPECTIONS,
at=date(2026, 7, 31),
)
next_month = claim_monthly_collection(
next_month = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.ARBITRATION,
at=date(2026, 8, 1),
@@ -46,8 +46,8 @@ class CheckoCollectionClaimTest(TestCase):
self.assertIsNotNone(other_source)
self.assertIsNotNone(next_month)
def test_failed_attempt_keeps_monthly_claim(self):
attempt = claim_monthly_collection(
def test_failed_attempt_keeps_weekly_claim(self):
attempt = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.CONTRACTS,
at=date(2026, 7, 19),
@@ -55,7 +55,7 @@ class CheckoCollectionClaimTest(TestCase):
assert attempt is not None
finish_collection(attempt, records_count=0, error="quota exhausted")
repeated = claim_monthly_collection(
repeated = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.CONTRACTS,
at=date(2026, 7, 20),
@@ -78,7 +78,7 @@ class CheckoCollectionClaimTest(TestCase):
)
second.opk_registry_membership = True
second.save(update_fields=["opk_registry_membership"])
claimed = claim_monthly_collection(
claimed = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.INSPECTIONS,
)
@@ -115,3 +115,15 @@ class CheckoCollectionClaimTest(TestCase):
[target.organization_id for target in targets],
[str(self.organization.id)],
)
def test_week_boundary_uses_moscow_saturday():
from datetime import datetime, timezone
from apps.parsers.checko_collection import collection_period_week
assert collection_period_week(
datetime(2026, 10, 30, 20, 59, tzinfo=timezone.utc)
) == date(2026, 10, 24)
assert collection_period_week(
datetime(2026, 10, 30, 21, 0, tzinfo=timezone.utc)
) == date(2026, 10, 31)

View File

@@ -6,7 +6,7 @@ from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch
import pytest
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.models import (
CheckoCollectionAttempt,
ParserSourceArtifact,
@@ -96,7 +96,7 @@ class CheckoSectionSnapshotTests(TestCase):
def test_claimed_organization_keeps_previous_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
claim_weekly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
assert self.existing(self.orgs[0]) == before
assert collector.call_count == 1
@@ -134,7 +134,7 @@ class CheckoSectionSnapshotTests(TestCase):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[1])
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
claim_weekly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
self.run_refresh(rows=[[]])
assert not self.existing(self.orgs[0])
assert self.existing(self.orgs[1]) == before
@@ -163,7 +163,7 @@ class CheckoSectionSnapshotTests(TestCase):
)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(
claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=SOURCE
)
is None
@@ -172,7 +172,7 @@ class CheckoSectionSnapshotTests(TestCase):
def test_no_completed_scope_rejects_without_publication(self):
for org in self.orgs:
self.seed(org)
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
before = [self.existing(org) for org in self.orgs]
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_refresh()
@@ -220,7 +220,7 @@ class CheckoSectionSnapshotTests(TestCase):
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
is None
)
@@ -246,7 +246,7 @@ class CheckoSectionSnapshotTests(TestCase):
before = [self.existing(org)[0][0] for org in self.orgs]
# Advance the calendar; never delete or reset a monthly claim.
with patch(
"apps.parsers.checko_collection.collection_period_month",
"apps.parsers.checko_collection.collection_period_week",
return_value=date(2026, 11, 1),
):
self.run_refresh(load_batch=2)
@@ -257,13 +257,13 @@ class CheckoSectionSnapshotTests(TestCase):
CheckoCollectionAttempt.Source.values
)
for source in adapter().CHECKO_SECTIONS:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
assert attempt is not None
finish_collection(attempt, records_count=0, error="fixture")
assert (
claim_monthly_collection(
claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
is None
@@ -311,7 +311,7 @@ def test_concurrent_monthly_claim_one_winner():
def claim():
return (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
is not None
)

View File

@@ -161,3 +161,41 @@ class RequestedSourceScheduleTest(TestCase):
migration.disable_schedules(apps, None)
assert not PeriodicTask.objects.filter(name__in=expected, enabled=True).exists()
assert PeriodicTask.objects.filter(name__in=expected).count() == 5
class WeeklySourcePolicyTest(TestCase):
def test_policy_consolidates_enabled_sources_and_preserves_disabled_gates(self):
migration = importlib.import_module(
"apps.parsers.migrations.0050_weekly_source_refresh"
)
interval = IntervalSchedule.objects.create(every=30, period="days")
source, _ = PeriodicTask.objects.update_or_create(
name="parser:checko-leasing:monthly",
defaults={
"task": "parsers.checko_leasing.refresh",
"enabled": True,
"interval": interval,
"crontab": None,
},
)
gated, _ = PeriodicTask.objects.update_or_create(
name="parser:sro-membership:full-msk",
defaults={
"task": "parsers.sro_membership_check.refresh",
"enabled": False,
"interval": interval,
},
)
migration.weekly_sources(apps, None)
migration.weekly_sources(apps, None)
source.refresh_from_db()
gated.refresh_from_db()
self.assertIsNone(source.interval_id)
self.assertEqual(source.crontab.day_of_week, "6")
self.assertEqual(str(source.crontab.timezone), "Europe/Moscow")
self.assertFalse(gated.enabled)
self.assertFalse(
PeriodicTask.objects.filter(
name__in=migration.DUPLICATES, enabled=True
).exists()
)

View File

@@ -324,7 +324,7 @@ class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase):
patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient),
patch.object(
parser_tasks,
"claim_monthly_collection",
"claim_weekly_collection",
return_value=SimpleNamespace(),
),
patch.object(parser_tasks, "finish_collection"),