fix: collect enabled sources weekly including Checko

This commit is contained in:
Aleksandr Meshchryakov
2026-10-29 17:46:26 +01:00
parent aeade84761
commit bc9767244d
12 changed files with 196 additions and 59 deletions

View File

@@ -1,34 +1,37 @@
"""Quota-safe monthly claims for external organization lookups."""
"""Quota-safe weekly claims for external organization lookups."""
from datetime import date, datetime
from datetime import date, datetime, timedelta
from zoneinfo import ZoneInfo
from apps.parsers.models import CheckoCollectionAttempt
from django.utils import timezone
def collection_period_month(at: date | datetime | None = None) -> date:
def collection_period_week(at: date | datetime | None = None) -> date:
if at is None:
local_date = timezone.localdate()
local_date = timezone.localdate(timezone=ZoneInfo("Europe/Moscow"))
elif isinstance(at, datetime):
local_date = (
timezone.localtime(at).date() if timezone.is_aware(at) else at.date()
timezone.localtime(at, ZoneInfo("Europe/Moscow")).date()
if timezone.is_aware(at)
else at.date()
)
else:
local_date = at
return local_date.replace(day=1)
return local_date - timedelta(days=(local_date.weekday() - 5) % 7)
def claim_monthly_collection(
def claim_weekly_collection(
*,
organization_id: str,
source: str,
at: date | datetime | None = None,
) -> CheckoCollectionAttempt | None:
"""Atomically claim this month's only allowed external collection attempt."""
"""Atomically claim this week's only allowed external collection attempt."""
attempt, created = CheckoCollectionAttempt.objects.get_or_create(
organization_id=organization_id,
source=source,
period_month=collection_period_month(at),
period_month=collection_period_week(at),
defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
)
return attempt if created else None
@@ -40,7 +43,7 @@ def finish_collection(
records_count: int,
error: Exception | str | None = None,
) -> None:
"""Persist the outcome without releasing the monthly claim."""
"""Persist the outcome without releasing the weekly claim."""
attempt.records_count = max(int(records_count), 0)
attempt.error_message = str(error or "")
attempt.status = (

View File

@@ -1,4 +1,4 @@
"""Monthly, canonical and scoped Checko collection; incomplete organizations survive."""
"""Weekly, canonical and scoped Checko collection; incomplete organizations survive."""
from __future__ import annotations
@@ -6,7 +6,7 @@ import json
import re
from tempfile import TemporaryFile
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
from apps.parsers.clients.checko import CheckoClient
from apps.parsers.fssp_csv import valid_legal_inn
@@ -143,7 +143,7 @@ def _collect_scope(client, organizations, source, artifact, raw_file, on_progres
counts["organizations_failed"] += 1
preserved.append(org.uid)
elif (
attempt := claim_monthly_collection(
attempt := claim_weekly_collection(
organization_id=str(org.uid), source=source
)
) is None:

View File

@@ -0,0 +1,61 @@
"""Collect enabled sources once a week on Saturday, Moscow time."""
from django.db import migrations, models
from django.utils import timezone
# Keep stable schedule IDs/names for existing clients and operational references.
EXTRA_SCHEDULES = (
"parser:fssp-enforcement-proceedings:daily",
"parser:stock-exchange-activity:daily",
"parser:checko-leasing:monthly",
"parser:checko-messages:monthly",
"parser:checko-licenses:monthly",
"parser:gosedo-address-directory:daily-msk",
"parser:registry-enrichment:daily-msk",
)
DUPLICATES = (
"parse-industrial-production-daily",
"parse-manufactures-daily",
"parse-industrial-products-daily",
)
def weekly_sources(apps, schema_editor):
Task = apps.get_model("django_celery_beat", "PeriodicTask")
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
cron, _ = Cron.objects.get_or_create(
minute="0",
hour="0",
day_of_week="6",
day_of_month="*",
month_of_year="*",
timezone="Europe/Moscow",
)
for task in Task.objects.filter(enabled=True):
if task.name in DUPLICATES:
task.enabled = False
task.save()
elif task.name in EXTRA_SCHEDULES or task.name.endswith(":weekly-saturday-msk"):
task.crontab = cron
task.interval = task.solar = task.clocked = None
task.description = "Weekly source collection: Saturday 00:00 Europe/Moscow."
task.save()
apps.get_model("django_celery_beat", "PeriodicTasks").objects.update_or_create(
ident=1,
defaults={"last_update": timezone.now()},
)
class Migration(migrations.Migration):
dependencies = [("parsers", "0049_aggregate_refresh_publication")]
operations = [
migrations.AlterField(
model_name="checkocollectionattempt",
name="period_month",
field=models.DateField(
"период сбора",
help_text="Начало недели сбора (суббота МСК); старые месячные попытки сохранены",
),
),
migrations.RunPython(weekly_sources, migrations.RunPython.noop),
]

View File

@@ -328,8 +328,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
choices=Source.choices,
)
period_month = models.DateField(
_("месяц сбора"),
help_text=_("Первый день календарного месяца, за который занята попытка"),
_("период сбора"),
help_text=_(
"Начало недели сбора (суббота МСК); старые месячные попытки сохранены"
),
)
status = models.CharField(
_("статус"),
@@ -356,7 +358,7 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
]
def __str__(self) -> str:
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})"
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m-%d})"
GENERIC_RECORD_SOURCE_CHOICES = [

View File

@@ -76,7 +76,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
data_scope="Лицензии организаций канонических организаций",
task_name="parsers.checko_licenses.refresh",
mode="organization_api",
access_method="monthly_claim_api",
access_method="weekly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
@@ -88,7 +88,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
data_scope="Сообщения организаций канонических организаций",
task_name="parsers.checko_messages.refresh",
mode="organization_api",
access_method="monthly_claim_api",
access_method="weekly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),
@@ -100,7 +100,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
data_scope="Лизинг канонических организаций",
task_name="parsers.checko_leasing.refresh",
mode="organization_api",
access_method="monthly_claim_api",
access_method="weekly_claim_api",
parser_strategy="scoped_organization_snapshot",
admin_only=True,
),

View File

@@ -21,8 +21,8 @@ from apps.core.services import BackgroundJobService
from apps.core.tasks import PeriodicTask as CorePeriodicTask
from apps.parsers.aggregate_publication import claim_part, finish_part
from apps.parsers.checko_collection import (
claim_monthly_collection,
collection_period_month,
claim_weekly_collection,
collection_period_week,
finish_collection,
)
from apps.parsers.clients.base import HTTPClientError
@@ -209,7 +209,7 @@ def _active_registry_lookup_targets(
if checko_source is not None:
queryset = queryset.exclude(
checko_collection_attempts__source=checko_source,
checko_collection_attempts__period_month=collection_period_month(),
checko_collection_attempts__period_month=collection_period_week(),
)
queryset = (
queryset.order_by("inn", "ogrn", "uid")
@@ -887,7 +887,7 @@ def _fetch_checko_bankruptcy_records(
)
if not targets:
logger.info(
"No active registry organizations are due for monthly Fedresurs fallback"
"No active registry organizations are due for weekly Fedresurs fallback"
)
return []
@@ -897,7 +897,7 @@ def _fetch_checko_bankruptcy_records(
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
records: list[GenericParserItem] = []
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.BANKRUPTCY,
)
@@ -1117,7 +1117,7 @@ def _fetch_checko_arbitration_records(
subjects = _arbitration_subjects(resolved_limit)
if not subjects:
raise ParserSourceSkipped(
"no active registry organizations are due for monthly arbitration lookup"
"no active registry organizations are due for weekly arbitration lookup"
)
checko_proxies = (
@@ -1128,7 +1128,7 @@ def _fetch_checko_arbitration_records(
failed_lookups = 0
attempted_lookups = 0
for subject in subjects:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=subject.registry_organization_id,
source=CheckoCollectionAttempt.Source.ARBITRATION,
)
@@ -1384,7 +1384,7 @@ def _fetch_checko_registry_inspections(
)
if not targets:
raise ParserSourceSkipped(
"no active registry organizations are due for monthly inspections lookup"
"no active registry organizations are due for weekly inspections lookup"
)
checko_proxies = (
@@ -1395,7 +1395,7 @@ def _fetch_checko_registry_inspections(
failed_lookups = 0
attempted_lookups = 0
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.INSPECTIONS,
)
@@ -1444,7 +1444,7 @@ def _fetch_eis_registry_procurement_records(
proxies: list[str] | None,
organization_ids: list[str] | None = None,
) -> list[GenericParserItem]:
"""Fetch addressed EIS notices for the next monthly OПК registry batch."""
"""Fetch addressed EIS notices for the next weekly OПК registry batch."""
resolved_limit = _resolve_registry_enrichment_limit(limit)
if resolved_limit <= 0:
return []
@@ -1461,7 +1461,7 @@ def _fetch_eis_registry_procurement_records(
)
if not targets:
raise ParserSourceSkipped(
f"no OПК registry organizations are due for monthly {law}-FZ lookup"
f"no OПК registry organizations are due for weekly {law}-FZ lookup"
)
client = EisRegistryProcurementClient(
@@ -1472,7 +1472,7 @@ def _fetch_eis_registry_procurement_records(
records: list[GenericParserItem] = []
failed_lookups = 0
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=claim_source,
)
@@ -1604,7 +1604,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
)
if not targets:
raise ParserSourceSkipped(
"no OПК registry organizations are due for monthly RNP lookup"
"no OПК registry organizations are due for weekly RNP lookup"
)
checko_proxies = (
@@ -1615,7 +1615,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
failed_lookups = 0
rate_limited = False
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
)
@@ -1780,7 +1780,7 @@ def _fetch_checko_registry_contract_records(
)
if not targets:
raise ParserSourceSkipped(
"no active registry organizations are due for monthly contracts lookup"
"no active registry organizations are due for weekly contracts lookup"
)
checko_proxies = (
@@ -1791,7 +1791,7 @@ def _fetch_checko_registry_contract_records(
failed_lookups = 0
attempted_lookups = 0
for target in targets:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=target.organization_id,
source=CheckoCollectionAttempt.Source.CONTRACTS,
)