fix: collect enabled sources weekly including Checko
This commit is contained in:
21
docs/weekly-source-refresh.md
Normal file
21
docs/weekly-source-refresh.md
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
# Недельное обновление источников
|
||||||
|
|
||||||
|
Сбор действующих источников запускается в субботу в 00:00 Europe/Moscow.
|
||||||
|
Миграция 0050 переводит дополнительные действующие расписания, включая три
|
||||||
|
раздела ЧЕКО, на этот cron и отключает три дублирующих interval-расписания.
|
||||||
|
Стабильные имена задач с суффиксами daily/monthly сохранены; фактический период
|
||||||
|
нужно читать из crontab. Отключённые источники с незавершённой приёмкой не включаются.
|
||||||
|
Обслуживание очередей, файлов и ежедневная сборка каталога выгрузок сохраняют свой период.
|
||||||
|
|
||||||
|
Защита ЧЕКО теперь разрешает одну попытку на организацию/раздел/неделю
|
||||||
|
(суббота–пятница МСК). Ошибка не освобождает попытку, параллельные запросы
|
||||||
|
защищены уникальным ограничением БД. Историческое имя колонки period_month
|
||||||
|
сохранено, её значение для новых попыток — дата субботы. Старые попытки не удаляются.
|
||||||
|
Недельный сбор может увеличить расход API относительно месячного.
|
||||||
|
|
||||||
|
Canonical OpenAPI 3 доступен по /openapi.json. Legacy /?format=openapi
|
||||||
|
возвращает Swagger 2 и не используется для генерации фронтенда.
|
||||||
|
|
||||||
|
После релиза проверить миграцию 0050, одинаковую ревизию web/worker/beat,
|
||||||
|
cron Europe/Moscow и отсутствие активных дублей; readiness должен вернуть 200.
|
||||||
|
Проверка регистрации расписания не доказывает успешный сбор с внешних источников.
|
||||||
@@ -1,34 +1,37 @@
|
|||||||
"""Quota-safe monthly claims for external organization lookups."""
|
"""Quota-safe weekly claims for external organization lookups."""
|
||||||
|
|
||||||
from datetime import date, datetime
|
from datetime import date, datetime, timedelta
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
from apps.parsers.models import CheckoCollectionAttempt
|
from apps.parsers.models import CheckoCollectionAttempt
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
|
||||||
|
|
||||||
def collection_period_month(at: date | datetime | None = None) -> date:
|
def collection_period_week(at: date | datetime | None = None) -> date:
|
||||||
if at is None:
|
if at is None:
|
||||||
local_date = timezone.localdate()
|
local_date = timezone.localdate(timezone=ZoneInfo("Europe/Moscow"))
|
||||||
elif isinstance(at, datetime):
|
elif isinstance(at, datetime):
|
||||||
local_date = (
|
local_date = (
|
||||||
timezone.localtime(at).date() if timezone.is_aware(at) else at.date()
|
timezone.localtime(at, ZoneInfo("Europe/Moscow")).date()
|
||||||
|
if timezone.is_aware(at)
|
||||||
|
else at.date()
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
local_date = at
|
local_date = at
|
||||||
return local_date.replace(day=1)
|
return local_date - timedelta(days=(local_date.weekday() - 5) % 7)
|
||||||
|
|
||||||
|
|
||||||
def claim_monthly_collection(
|
def claim_weekly_collection(
|
||||||
*,
|
*,
|
||||||
organization_id: str,
|
organization_id: str,
|
||||||
source: str,
|
source: str,
|
||||||
at: date | datetime | None = None,
|
at: date | datetime | None = None,
|
||||||
) -> CheckoCollectionAttempt | None:
|
) -> CheckoCollectionAttempt | None:
|
||||||
"""Atomically claim this month's only allowed external collection attempt."""
|
"""Atomically claim this week's only allowed external collection attempt."""
|
||||||
attempt, created = CheckoCollectionAttempt.objects.get_or_create(
|
attempt, created = CheckoCollectionAttempt.objects.get_or_create(
|
||||||
organization_id=organization_id,
|
organization_id=organization_id,
|
||||||
source=source,
|
source=source,
|
||||||
period_month=collection_period_month(at),
|
period_month=collection_period_week(at),
|
||||||
defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
|
defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
|
||||||
)
|
)
|
||||||
return attempt if created else None
|
return attempt if created else None
|
||||||
@@ -40,7 +43,7 @@ def finish_collection(
|
|||||||
records_count: int,
|
records_count: int,
|
||||||
error: Exception | str | None = None,
|
error: Exception | str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Persist the outcome without releasing the monthly claim."""
|
"""Persist the outcome without releasing the weekly claim."""
|
||||||
attempt.records_count = max(int(records_count), 0)
|
attempt.records_count = max(int(records_count), 0)
|
||||||
attempt.error_message = str(error or "")
|
attempt.error_message = str(error or "")
|
||||||
attempt.status = (
|
attempt.status = (
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Monthly, canonical and scoped Checko collection; incomplete organizations survive."""
|
"""Weekly, canonical and scoped Checko collection; incomplete organizations survive."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -6,7 +6,7 @@ import json
|
|||||||
import re
|
import re
|
||||||
from tempfile import TemporaryFile
|
from tempfile import TemporaryFile
|
||||||
|
|
||||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||||
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
|
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
|
||||||
from apps.parsers.clients.checko import CheckoClient
|
from apps.parsers.clients.checko import CheckoClient
|
||||||
from apps.parsers.fssp_csv import valid_legal_inn
|
from apps.parsers.fssp_csv import valid_legal_inn
|
||||||
@@ -143,7 +143,7 @@ def _collect_scope(client, organizations, source, artifact, raw_file, on_progres
|
|||||||
counts["organizations_failed"] += 1
|
counts["organizations_failed"] += 1
|
||||||
preserved.append(org.uid)
|
preserved.append(org.uid)
|
||||||
elif (
|
elif (
|
||||||
attempt := claim_monthly_collection(
|
attempt := claim_weekly_collection(
|
||||||
organization_id=str(org.uid), source=source
|
organization_id=str(org.uid), source=source
|
||||||
)
|
)
|
||||||
) is None:
|
) is None:
|
||||||
|
|||||||
61
src/apps/parsers/migrations/0050_weekly_source_refresh.py
Normal file
61
src/apps/parsers/migrations/0050_weekly_source_refresh.py
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
"""Collect enabled sources once a week on Saturday, Moscow time."""
|
||||||
|
|
||||||
|
from django.db import migrations, models
|
||||||
|
from django.utils import timezone
|
||||||
|
|
||||||
|
# Keep stable schedule IDs/names for existing clients and operational references.
|
||||||
|
EXTRA_SCHEDULES = (
|
||||||
|
"parser:fssp-enforcement-proceedings:daily",
|
||||||
|
"parser:stock-exchange-activity:daily",
|
||||||
|
"parser:checko-leasing:monthly",
|
||||||
|
"parser:checko-messages:monthly",
|
||||||
|
"parser:checko-licenses:monthly",
|
||||||
|
"parser:gosedo-address-directory:daily-msk",
|
||||||
|
"parser:registry-enrichment:daily-msk",
|
||||||
|
)
|
||||||
|
DUPLICATES = (
|
||||||
|
"parse-industrial-production-daily",
|
||||||
|
"parse-manufactures-daily",
|
||||||
|
"parse-industrial-products-daily",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def weekly_sources(apps, schema_editor):
|
||||||
|
Task = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||||
|
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
|
||||||
|
cron, _ = Cron.objects.get_or_create(
|
||||||
|
minute="0",
|
||||||
|
hour="0",
|
||||||
|
day_of_week="6",
|
||||||
|
day_of_month="*",
|
||||||
|
month_of_year="*",
|
||||||
|
timezone="Europe/Moscow",
|
||||||
|
)
|
||||||
|
for task in Task.objects.filter(enabled=True):
|
||||||
|
if task.name in DUPLICATES:
|
||||||
|
task.enabled = False
|
||||||
|
task.save()
|
||||||
|
elif task.name in EXTRA_SCHEDULES or task.name.endswith(":weekly-saturday-msk"):
|
||||||
|
task.crontab = cron
|
||||||
|
task.interval = task.solar = task.clocked = None
|
||||||
|
task.description = "Weekly source collection: Saturday 00:00 Europe/Moscow."
|
||||||
|
task.save()
|
||||||
|
apps.get_model("django_celery_beat", "PeriodicTasks").objects.update_or_create(
|
||||||
|
ident=1,
|
||||||
|
defaults={"last_update": timezone.now()},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [("parsers", "0049_aggregate_refresh_publication")]
|
||||||
|
operations = [
|
||||||
|
migrations.AlterField(
|
||||||
|
model_name="checkocollectionattempt",
|
||||||
|
name="period_month",
|
||||||
|
field=models.DateField(
|
||||||
|
"период сбора",
|
||||||
|
help_text="Начало недели сбора (суббота МСК); старые месячные попытки сохранены",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
migrations.RunPython(weekly_sources, migrations.RunPython.noop),
|
||||||
|
]
|
||||||
@@ -328,8 +328,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
|||||||
choices=Source.choices,
|
choices=Source.choices,
|
||||||
)
|
)
|
||||||
period_month = models.DateField(
|
period_month = models.DateField(
|
||||||
_("месяц сбора"),
|
_("период сбора"),
|
||||||
help_text=_("Первый день календарного месяца, за который занята попытка"),
|
help_text=_(
|
||||||
|
"Начало недели сбора (суббота МСК); старые месячные попытки сохранены"
|
||||||
|
),
|
||||||
)
|
)
|
||||||
status = models.CharField(
|
status = models.CharField(
|
||||||
_("статус"),
|
_("статус"),
|
||||||
@@ -356,7 +358,7 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
|||||||
]
|
]
|
||||||
|
|
||||||
def __str__(self) -> str:
|
def __str__(self) -> str:
|
||||||
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})"
|
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m-%d})"
|
||||||
|
|
||||||
|
|
||||||
GENERIC_RECORD_SOURCE_CHOICES = [
|
GENERIC_RECORD_SOURCE_CHOICES = [
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
|||||||
data_scope="Лицензии организаций канонических организаций",
|
data_scope="Лицензии организаций канонических организаций",
|
||||||
task_name="parsers.checko_licenses.refresh",
|
task_name="parsers.checko_licenses.refresh",
|
||||||
mode="organization_api",
|
mode="organization_api",
|
||||||
access_method="monthly_claim_api",
|
access_method="weekly_claim_api",
|
||||||
parser_strategy="scoped_organization_snapshot",
|
parser_strategy="scoped_organization_snapshot",
|
||||||
admin_only=True,
|
admin_only=True,
|
||||||
),
|
),
|
||||||
@@ -88,7 +88,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
|||||||
data_scope="Сообщения организаций канонических организаций",
|
data_scope="Сообщения организаций канонических организаций",
|
||||||
task_name="parsers.checko_messages.refresh",
|
task_name="parsers.checko_messages.refresh",
|
||||||
mode="organization_api",
|
mode="organization_api",
|
||||||
access_method="monthly_claim_api",
|
access_method="weekly_claim_api",
|
||||||
parser_strategy="scoped_organization_snapshot",
|
parser_strategy="scoped_organization_snapshot",
|
||||||
admin_only=True,
|
admin_only=True,
|
||||||
),
|
),
|
||||||
@@ -100,7 +100,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
|||||||
data_scope="Лизинг канонических организаций",
|
data_scope="Лизинг канонических организаций",
|
||||||
task_name="parsers.checko_leasing.refresh",
|
task_name="parsers.checko_leasing.refresh",
|
||||||
mode="organization_api",
|
mode="organization_api",
|
||||||
access_method="monthly_claim_api",
|
access_method="weekly_claim_api",
|
||||||
parser_strategy="scoped_organization_snapshot",
|
parser_strategy="scoped_organization_snapshot",
|
||||||
admin_only=True,
|
admin_only=True,
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ from apps.core.services import BackgroundJobService
|
|||||||
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
||||||
from apps.parsers.aggregate_publication import claim_part, finish_part
|
from apps.parsers.aggregate_publication import claim_part, finish_part
|
||||||
from apps.parsers.checko_collection import (
|
from apps.parsers.checko_collection import (
|
||||||
claim_monthly_collection,
|
claim_weekly_collection,
|
||||||
collection_period_month,
|
collection_period_week,
|
||||||
finish_collection,
|
finish_collection,
|
||||||
)
|
)
|
||||||
from apps.parsers.clients.base import HTTPClientError
|
from apps.parsers.clients.base import HTTPClientError
|
||||||
@@ -209,7 +209,7 @@ def _active_registry_lookup_targets(
|
|||||||
if checko_source is not None:
|
if checko_source is not None:
|
||||||
queryset = queryset.exclude(
|
queryset = queryset.exclude(
|
||||||
checko_collection_attempts__source=checko_source,
|
checko_collection_attempts__source=checko_source,
|
||||||
checko_collection_attempts__period_month=collection_period_month(),
|
checko_collection_attempts__period_month=collection_period_week(),
|
||||||
)
|
)
|
||||||
queryset = (
|
queryset = (
|
||||||
queryset.order_by("inn", "ogrn", "uid")
|
queryset.order_by("inn", "ogrn", "uid")
|
||||||
@@ -887,7 +887,7 @@ def _fetch_checko_bankruptcy_records(
|
|||||||
)
|
)
|
||||||
if not targets:
|
if not targets:
|
||||||
logger.info(
|
logger.info(
|
||||||
"No active registry organizations are due for monthly Fedresurs fallback"
|
"No active registry organizations are due for weekly Fedresurs fallback"
|
||||||
)
|
)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@@ -897,7 +897,7 @@ def _fetch_checko_bankruptcy_records(
|
|||||||
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
|
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
|
||||||
records: list[GenericParserItem] = []
|
records: list[GenericParserItem] = []
|
||||||
for target in targets:
|
for target in targets:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=target.organization_id,
|
organization_id=target.organization_id,
|
||||||
source=CheckoCollectionAttempt.Source.BANKRUPTCY,
|
source=CheckoCollectionAttempt.Source.BANKRUPTCY,
|
||||||
)
|
)
|
||||||
@@ -1117,7 +1117,7 @@ def _fetch_checko_arbitration_records(
|
|||||||
subjects = _arbitration_subjects(resolved_limit)
|
subjects = _arbitration_subjects(resolved_limit)
|
||||||
if not subjects:
|
if not subjects:
|
||||||
raise ParserSourceSkipped(
|
raise ParserSourceSkipped(
|
||||||
"no active registry organizations are due for monthly arbitration lookup"
|
"no active registry organizations are due for weekly arbitration lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
checko_proxies = (
|
checko_proxies = (
|
||||||
@@ -1128,7 +1128,7 @@ def _fetch_checko_arbitration_records(
|
|||||||
failed_lookups = 0
|
failed_lookups = 0
|
||||||
attempted_lookups = 0
|
attempted_lookups = 0
|
||||||
for subject in subjects:
|
for subject in subjects:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=subject.registry_organization_id,
|
organization_id=subject.registry_organization_id,
|
||||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||||
)
|
)
|
||||||
@@ -1384,7 +1384,7 @@ def _fetch_checko_registry_inspections(
|
|||||||
)
|
)
|
||||||
if not targets:
|
if not targets:
|
||||||
raise ParserSourceSkipped(
|
raise ParserSourceSkipped(
|
||||||
"no active registry organizations are due for monthly inspections lookup"
|
"no active registry organizations are due for weekly inspections lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
checko_proxies = (
|
checko_proxies = (
|
||||||
@@ -1395,7 +1395,7 @@ def _fetch_checko_registry_inspections(
|
|||||||
failed_lookups = 0
|
failed_lookups = 0
|
||||||
attempted_lookups = 0
|
attempted_lookups = 0
|
||||||
for target in targets:
|
for target in targets:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=target.organization_id,
|
organization_id=target.organization_id,
|
||||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||||
)
|
)
|
||||||
@@ -1444,7 +1444,7 @@ def _fetch_eis_registry_procurement_records(
|
|||||||
proxies: list[str] | None,
|
proxies: list[str] | None,
|
||||||
organization_ids: list[str] | None = None,
|
organization_ids: list[str] | None = None,
|
||||||
) -> list[GenericParserItem]:
|
) -> list[GenericParserItem]:
|
||||||
"""Fetch addressed EIS notices for the next monthly OПК registry batch."""
|
"""Fetch addressed EIS notices for the next weekly OПК registry batch."""
|
||||||
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
||||||
if resolved_limit <= 0:
|
if resolved_limit <= 0:
|
||||||
return []
|
return []
|
||||||
@@ -1461,7 +1461,7 @@ def _fetch_eis_registry_procurement_records(
|
|||||||
)
|
)
|
||||||
if not targets:
|
if not targets:
|
||||||
raise ParserSourceSkipped(
|
raise ParserSourceSkipped(
|
||||||
f"no OПК registry organizations are due for monthly {law}-FZ lookup"
|
f"no OПК registry organizations are due for weekly {law}-FZ lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
client = EisRegistryProcurementClient(
|
client = EisRegistryProcurementClient(
|
||||||
@@ -1472,7 +1472,7 @@ def _fetch_eis_registry_procurement_records(
|
|||||||
records: list[GenericParserItem] = []
|
records: list[GenericParserItem] = []
|
||||||
failed_lookups = 0
|
failed_lookups = 0
|
||||||
for target in targets:
|
for target in targets:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=target.organization_id,
|
organization_id=target.organization_id,
|
||||||
source=claim_source,
|
source=claim_source,
|
||||||
)
|
)
|
||||||
@@ -1604,7 +1604,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
|
|||||||
)
|
)
|
||||||
if not targets:
|
if not targets:
|
||||||
raise ParserSourceSkipped(
|
raise ParserSourceSkipped(
|
||||||
"no OПК registry organizations are due for monthly RNP lookup"
|
"no OПК registry organizations are due for weekly RNP lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
checko_proxies = (
|
checko_proxies = (
|
||||||
@@ -1615,7 +1615,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
|
|||||||
failed_lookups = 0
|
failed_lookups = 0
|
||||||
rate_limited = False
|
rate_limited = False
|
||||||
for target in targets:
|
for target in targets:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=target.organization_id,
|
organization_id=target.organization_id,
|
||||||
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
|
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
|
||||||
)
|
)
|
||||||
@@ -1780,7 +1780,7 @@ def _fetch_checko_registry_contract_records(
|
|||||||
)
|
)
|
||||||
if not targets:
|
if not targets:
|
||||||
raise ParserSourceSkipped(
|
raise ParserSourceSkipped(
|
||||||
"no active registry organizations are due for monthly contracts lookup"
|
"no active registry organizations are due for weekly contracts lookup"
|
||||||
)
|
)
|
||||||
|
|
||||||
checko_proxies = (
|
checko_proxies = (
|
||||||
@@ -1791,7 +1791,7 @@ def _fetch_checko_registry_contract_records(
|
|||||||
failed_lookups = 0
|
failed_lookups = 0
|
||||||
attempted_lookups = 0
|
attempted_lookups = 0
|
||||||
for target in targets:
|
for target in targets:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=target.organization_id,
|
organization_id=target.organization_id,
|
||||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from unittest.mock import patch
|
|||||||
import pytest
|
import pytest
|
||||||
from apps.core.models import BackgroundJob
|
from apps.core.models import BackgroundJob
|
||||||
from apps.parsers import checko_section_snapshots as snapshots
|
from apps.parsers import checko_section_snapshots as snapshots
|
||||||
from apps.parsers.checko_collection import claim_monthly_collection
|
from apps.parsers.checko_collection import claim_weekly_collection
|
||||||
from apps.parsers.models import ParserSourceArtifact
|
from apps.parsers.models import ParserSourceArtifact
|
||||||
from apps.parsers.registry_snapshots import SnapshotValidationError
|
from apps.parsers.registry_snapshots import SnapshotValidationError
|
||||||
from apps.parsers.source_cards import SourceCardService
|
from apps.parsers.source_cards import SourceCardService
|
||||||
@@ -174,7 +174,7 @@ class CheckoSectionContractTests(TestCase):
|
|||||||
|
|
||||||
def test_no_completed_failure_exposes_rejected_counters(self):
|
def test_no_completed_failure_exposes_rejected_counters(self):
|
||||||
source = "checko_messages"
|
source = "checko_messages"
|
||||||
claim_monthly_collection(organization_id=str(self.org.uid), source=source)
|
claim_weekly_collection(organization_id=str(self.org.uid), source=source)
|
||||||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||||||
self.run_task(source)
|
self.run_task(source)
|
||||||
job = BackgroundJob.objects.get()
|
job = BackgroundJob.objects.get()
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
from datetime import date
|
from datetime import date
|
||||||
|
|
||||||
from apps.parsers import tasks as parser_tasks
|
from apps.parsers import tasks as parser_tasks
|
||||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||||
from apps.parsers.models import CheckoCollectionAttempt
|
from apps.parsers.models import CheckoCollectionAttempt
|
||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
|
|
||||||
@@ -18,24 +18,24 @@ class CheckoCollectionClaimTest(TestCase):
|
|||||||
mn_okpo="12345678",
|
mn_okpo="12345678",
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_claim_is_unique_per_organization_source_and_calendar_month(self):
|
def test_claim_is_unique_per_organization_source_and_calendar_week(self):
|
||||||
first = claim_monthly_collection(
|
first = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||||
at=date(2026, 7, 1),
|
at=date(2026, 7, 25),
|
||||||
)
|
)
|
||||||
|
|
||||||
duplicate = claim_monthly_collection(
|
duplicate = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||||
at=date(2026, 7, 31),
|
at=date(2026, 7, 31),
|
||||||
)
|
)
|
||||||
other_source = claim_monthly_collection(
|
other_source = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||||
at=date(2026, 7, 31),
|
at=date(2026, 7, 31),
|
||||||
)
|
)
|
||||||
next_month = claim_monthly_collection(
|
next_month = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||||
at=date(2026, 8, 1),
|
at=date(2026, 8, 1),
|
||||||
@@ -46,8 +46,8 @@ class CheckoCollectionClaimTest(TestCase):
|
|||||||
self.assertIsNotNone(other_source)
|
self.assertIsNotNone(other_source)
|
||||||
self.assertIsNotNone(next_month)
|
self.assertIsNotNone(next_month)
|
||||||
|
|
||||||
def test_failed_attempt_keeps_monthly_claim(self):
|
def test_failed_attempt_keeps_weekly_claim(self):
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||||
at=date(2026, 7, 19),
|
at=date(2026, 7, 19),
|
||||||
@@ -55,7 +55,7 @@ class CheckoCollectionClaimTest(TestCase):
|
|||||||
assert attempt is not None
|
assert attempt is not None
|
||||||
finish_collection(attempt, records_count=0, error="quota exhausted")
|
finish_collection(attempt, records_count=0, error="quota exhausted")
|
||||||
|
|
||||||
repeated = claim_monthly_collection(
|
repeated = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||||
at=date(2026, 7, 20),
|
at=date(2026, 7, 20),
|
||||||
@@ -78,7 +78,7 @@ class CheckoCollectionClaimTest(TestCase):
|
|||||||
)
|
)
|
||||||
second.opk_registry_membership = True
|
second.opk_registry_membership = True
|
||||||
second.save(update_fields=["opk_registry_membership"])
|
second.save(update_fields=["opk_registry_membership"])
|
||||||
claimed = claim_monthly_collection(
|
claimed = claim_weekly_collection(
|
||||||
organization_id=str(self.organization.id),
|
organization_id=str(self.organization.id),
|
||||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||||
)
|
)
|
||||||
@@ -115,3 +115,15 @@ class CheckoCollectionClaimTest(TestCase):
|
|||||||
[target.organization_id for target in targets],
|
[target.organization_id for target in targets],
|
||||||
[str(self.organization.id)],
|
[str(self.organization.id)],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_week_boundary_uses_moscow_saturday():
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from apps.parsers.checko_collection import collection_period_week
|
||||||
|
|
||||||
|
assert collection_period_week(
|
||||||
|
datetime(2026, 10, 30, 20, 59, tzinfo=timezone.utc)
|
||||||
|
) == date(2026, 10, 24)
|
||||||
|
assert collection_period_week(
|
||||||
|
datetime(2026, 10, 30, 21, 0, tzinfo=timezone.utc)
|
||||||
|
) == date(2026, 10, 31)
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from tempfile import TemporaryDirectory
|
|||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||||
from apps.parsers.models import (
|
from apps.parsers.models import (
|
||||||
CheckoCollectionAttempt,
|
CheckoCollectionAttempt,
|
||||||
ParserSourceArtifact,
|
ParserSourceArtifact,
|
||||||
@@ -96,7 +96,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
def test_claimed_organization_keeps_previous_records(self):
|
def test_claimed_organization_keeps_previous_records(self):
|
||||||
self.seed(self.orgs[0])
|
self.seed(self.orgs[0])
|
||||||
before = self.existing(self.orgs[0])
|
before = self.existing(self.orgs[0])
|
||||||
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
|
claim_weekly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
|
||||||
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
|
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
|
||||||
assert self.existing(self.orgs[0]) == before
|
assert self.existing(self.orgs[0]) == before
|
||||||
assert collector.call_count == 1
|
assert collector.call_count == 1
|
||||||
@@ -134,7 +134,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
for org in self.orgs:
|
for org in self.orgs:
|
||||||
self.seed(org)
|
self.seed(org)
|
||||||
before = self.existing(self.orgs[1])
|
before = self.existing(self.orgs[1])
|
||||||
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
|
claim_weekly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
|
||||||
self.run_refresh(rows=[[]])
|
self.run_refresh(rows=[[]])
|
||||||
assert not self.existing(self.orgs[0])
|
assert not self.existing(self.orgs[0])
|
||||||
assert self.existing(self.orgs[1]) == before
|
assert self.existing(self.orgs[1]) == before
|
||||||
@@ -163,7 +163,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
)
|
)
|
||||||
assert attempt.status == attempt.Status.FAILED
|
assert attempt.status == attempt.Status.FAILED
|
||||||
assert (
|
assert (
|
||||||
claim_monthly_collection(
|
claim_weekly_collection(
|
||||||
organization_id=str(self.orgs[0].uid), source=SOURCE
|
organization_id=str(self.orgs[0].uid), source=SOURCE
|
||||||
)
|
)
|
||||||
is None
|
is None
|
||||||
@@ -172,7 +172,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
def test_no_completed_scope_rejects_without_publication(self):
|
def test_no_completed_scope_rejects_without_publication(self):
|
||||||
for org in self.orgs:
|
for org in self.orgs:
|
||||||
self.seed(org)
|
self.seed(org)
|
||||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||||
before = [self.existing(org) for org in self.orgs]
|
before = [self.existing(org) for org in self.orgs]
|
||||||
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
|
||||||
self.run_refresh()
|
self.run_refresh()
|
||||||
@@ -220,7 +220,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
|
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
|
||||||
assert attempt.status == attempt.Status.FAILED
|
assert attempt.status == attempt.Status.FAILED
|
||||||
assert (
|
assert (
|
||||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||||
is None
|
is None
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -246,7 +246,7 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
before = [self.existing(org)[0][0] for org in self.orgs]
|
before = [self.existing(org)[0][0] for org in self.orgs]
|
||||||
# Advance the calendar; never delete or reset a monthly claim.
|
# Advance the calendar; never delete or reset a monthly claim.
|
||||||
with patch(
|
with patch(
|
||||||
"apps.parsers.checko_collection.collection_period_month",
|
"apps.parsers.checko_collection.collection_period_week",
|
||||||
return_value=date(2026, 11, 1),
|
return_value=date(2026, 11, 1),
|
||||||
):
|
):
|
||||||
self.run_refresh(load_batch=2)
|
self.run_refresh(load_batch=2)
|
||||||
@@ -257,13 +257,13 @@ class CheckoSectionSnapshotTests(TestCase):
|
|||||||
CheckoCollectionAttempt.Source.values
|
CheckoCollectionAttempt.Source.values
|
||||||
)
|
)
|
||||||
for source in adapter().CHECKO_SECTIONS:
|
for source in adapter().CHECKO_SECTIONS:
|
||||||
attempt = claim_monthly_collection(
|
attempt = claim_weekly_collection(
|
||||||
organization_id=str(self.orgs[0].uid), source=source
|
organization_id=str(self.orgs[0].uid), source=source
|
||||||
)
|
)
|
||||||
assert attempt is not None
|
assert attempt is not None
|
||||||
finish_collection(attempt, records_count=0, error="fixture")
|
finish_collection(attempt, records_count=0, error="fixture")
|
||||||
assert (
|
assert (
|
||||||
claim_monthly_collection(
|
claim_weekly_collection(
|
||||||
organization_id=str(self.orgs[0].uid), source=source
|
organization_id=str(self.orgs[0].uid), source=source
|
||||||
)
|
)
|
||||||
is None
|
is None
|
||||||
@@ -311,7 +311,7 @@ def test_concurrent_monthly_claim_one_winner():
|
|||||||
|
|
||||||
def claim():
|
def claim():
|
||||||
return (
|
return (
|
||||||
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
|
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
|
||||||
is not None
|
is not None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -161,3 +161,41 @@ class RequestedSourceScheduleTest(TestCase):
|
|||||||
migration.disable_schedules(apps, None)
|
migration.disable_schedules(apps, None)
|
||||||
assert not PeriodicTask.objects.filter(name__in=expected, enabled=True).exists()
|
assert not PeriodicTask.objects.filter(name__in=expected, enabled=True).exists()
|
||||||
assert PeriodicTask.objects.filter(name__in=expected).count() == 5
|
assert PeriodicTask.objects.filter(name__in=expected).count() == 5
|
||||||
|
|
||||||
|
|
||||||
|
class WeeklySourcePolicyTest(TestCase):
|
||||||
|
def test_policy_consolidates_enabled_sources_and_preserves_disabled_gates(self):
|
||||||
|
migration = importlib.import_module(
|
||||||
|
"apps.parsers.migrations.0050_weekly_source_refresh"
|
||||||
|
)
|
||||||
|
interval = IntervalSchedule.objects.create(every=30, period="days")
|
||||||
|
source, _ = PeriodicTask.objects.update_or_create(
|
||||||
|
name="parser:checko-leasing:monthly",
|
||||||
|
defaults={
|
||||||
|
"task": "parsers.checko_leasing.refresh",
|
||||||
|
"enabled": True,
|
||||||
|
"interval": interval,
|
||||||
|
"crontab": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
gated, _ = PeriodicTask.objects.update_or_create(
|
||||||
|
name="parser:sro-membership:full-msk",
|
||||||
|
defaults={
|
||||||
|
"task": "parsers.sro_membership_check.refresh",
|
||||||
|
"enabled": False,
|
||||||
|
"interval": interval,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
migration.weekly_sources(apps, None)
|
||||||
|
migration.weekly_sources(apps, None)
|
||||||
|
source.refresh_from_db()
|
||||||
|
gated.refresh_from_db()
|
||||||
|
self.assertIsNone(source.interval_id)
|
||||||
|
self.assertEqual(source.crontab.day_of_week, "6")
|
||||||
|
self.assertEqual(str(source.crontab.timezone), "Europe/Moscow")
|
||||||
|
self.assertFalse(gated.enabled)
|
||||||
|
self.assertFalse(
|
||||||
|
PeriodicTask.objects.filter(
|
||||||
|
name__in=migration.DUPLICATES, enabled=True
|
||||||
|
).exists()
|
||||||
|
)
|
||||||
|
|||||||
@@ -324,7 +324,7 @@ class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase):
|
|||||||
patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient),
|
patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient),
|
||||||
patch.object(
|
patch.object(
|
||||||
parser_tasks,
|
parser_tasks,
|
||||||
"claim_monthly_collection",
|
"claim_weekly_collection",
|
||||||
return_value=SimpleNamespace(),
|
return_value=SimpleNamespace(),
|
||||||
),
|
),
|
||||||
patch.object(parser_tasks, "finish_collection"),
|
patch.object(parser_tasks, "finish_collection"),
|
||||||
|
|||||||
Reference in New Issue
Block a user