fix: collect enabled sources weekly including Checko
This commit is contained in:
@@ -1,34 +1,37 @@
|
||||
"""Quota-safe monthly claims for external organization lookups."""
|
||||
"""Quota-safe weekly claims for external organization lookups."""
|
||||
|
||||
from datetime import date, datetime
|
||||
from datetime import date, datetime, timedelta
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from apps.parsers.models import CheckoCollectionAttempt
|
||||
from django.utils import timezone
|
||||
|
||||
|
||||
def collection_period_month(at: date | datetime | None = None) -> date:
|
||||
def collection_period_week(at: date | datetime | None = None) -> date:
|
||||
if at is None:
|
||||
local_date = timezone.localdate()
|
||||
local_date = timezone.localdate(timezone=ZoneInfo("Europe/Moscow"))
|
||||
elif isinstance(at, datetime):
|
||||
local_date = (
|
||||
timezone.localtime(at).date() if timezone.is_aware(at) else at.date()
|
||||
timezone.localtime(at, ZoneInfo("Europe/Moscow")).date()
|
||||
if timezone.is_aware(at)
|
||||
else at.date()
|
||||
)
|
||||
else:
|
||||
local_date = at
|
||||
return local_date.replace(day=1)
|
||||
return local_date - timedelta(days=(local_date.weekday() - 5) % 7)
|
||||
|
||||
|
||||
def claim_monthly_collection(
|
||||
def claim_weekly_collection(
|
||||
*,
|
||||
organization_id: str,
|
||||
source: str,
|
||||
at: date | datetime | None = None,
|
||||
) -> CheckoCollectionAttempt | None:
|
||||
"""Atomically claim this month's only allowed external collection attempt."""
|
||||
"""Atomically claim this week's only allowed external collection attempt."""
|
||||
attempt, created = CheckoCollectionAttempt.objects.get_or_create(
|
||||
organization_id=organization_id,
|
||||
source=source,
|
||||
period_month=collection_period_month(at),
|
||||
period_month=collection_period_week(at),
|
||||
defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
|
||||
)
|
||||
return attempt if created else None
|
||||
@@ -40,7 +43,7 @@ def finish_collection(
|
||||
records_count: int,
|
||||
error: Exception | str | None = None,
|
||||
) -> None:
|
||||
"""Persist the outcome without releasing the monthly claim."""
|
||||
"""Persist the outcome without releasing the weekly claim."""
|
||||
attempt.records_count = max(int(records_count), 0)
|
||||
attempt.error_message = str(error or "")
|
||||
attempt.status = (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Monthly, canonical and scoped Checko collection; incomplete organizations survive."""
|
||||
"""Weekly, canonical and scoped Checko collection; incomplete organizations survive."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import re
|
||||
from tempfile import TemporaryFile
|
||||
|
||||
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
|
||||
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
|
||||
from apps.parsers.checko_sections import CHECKO_SECTIONS, collect_checko_section
|
||||
from apps.parsers.clients.checko import CheckoClient
|
||||
from apps.parsers.fssp_csv import valid_legal_inn
|
||||
@@ -143,7 +143,7 @@ def _collect_scope(client, organizations, source, artifact, raw_file, on_progres
|
||||
counts["organizations_failed"] += 1
|
||||
preserved.append(org.uid)
|
||||
elif (
|
||||
attempt := claim_monthly_collection(
|
||||
attempt := claim_weekly_collection(
|
||||
organization_id=str(org.uid), source=source
|
||||
)
|
||||
) is None:
|
||||
|
||||
61
src/apps/parsers/migrations/0050_weekly_source_refresh.py
Normal file
61
src/apps/parsers/migrations/0050_weekly_source_refresh.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Collect enabled sources once a week on Saturday, Moscow time."""
|
||||
|
||||
from django.db import migrations, models
|
||||
from django.utils import timezone
|
||||
|
||||
# Keep stable schedule IDs/names for existing clients and operational references.
|
||||
EXTRA_SCHEDULES = (
|
||||
"parser:fssp-enforcement-proceedings:daily",
|
||||
"parser:stock-exchange-activity:daily",
|
||||
"parser:checko-leasing:monthly",
|
||||
"parser:checko-messages:monthly",
|
||||
"parser:checko-licenses:monthly",
|
||||
"parser:gosedo-address-directory:daily-msk",
|
||||
"parser:registry-enrichment:daily-msk",
|
||||
)
|
||||
DUPLICATES = (
|
||||
"parse-industrial-production-daily",
|
||||
"parse-manufactures-daily",
|
||||
"parse-industrial-products-daily",
|
||||
)
|
||||
|
||||
|
||||
def weekly_sources(apps, schema_editor):
|
||||
Task = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
Cron = apps.get_model("django_celery_beat", "CrontabSchedule")
|
||||
cron, _ = Cron.objects.get_or_create(
|
||||
minute="0",
|
||||
hour="0",
|
||||
day_of_week="6",
|
||||
day_of_month="*",
|
||||
month_of_year="*",
|
||||
timezone="Europe/Moscow",
|
||||
)
|
||||
for task in Task.objects.filter(enabled=True):
|
||||
if task.name in DUPLICATES:
|
||||
task.enabled = False
|
||||
task.save()
|
||||
elif task.name in EXTRA_SCHEDULES or task.name.endswith(":weekly-saturday-msk"):
|
||||
task.crontab = cron
|
||||
task.interval = task.solar = task.clocked = None
|
||||
task.description = "Weekly source collection: Saturday 00:00 Europe/Moscow."
|
||||
task.save()
|
||||
apps.get_model("django_celery_beat", "PeriodicTasks").objects.update_or_create(
|
||||
ident=1,
|
||||
defaults={"last_update": timezone.now()},
|
||||
)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [("parsers", "0049_aggregate_refresh_publication")]
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="checkocollectionattempt",
|
||||
name="period_month",
|
||||
field=models.DateField(
|
||||
"период сбора",
|
||||
help_text="Начало недели сбора (суббота МСК); старые месячные попытки сохранены",
|
||||
),
|
||||
),
|
||||
migrations.RunPython(weekly_sources, migrations.RunPython.noop),
|
||||
]
|
||||
@@ -328,8 +328,10 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
||||
choices=Source.choices,
|
||||
)
|
||||
period_month = models.DateField(
|
||||
_("месяц сбора"),
|
||||
help_text=_("Первый день календарного месяца, за который занята попытка"),
|
||||
_("период сбора"),
|
||||
help_text=_(
|
||||
"Начало недели сбора (суббота МСК); старые месячные попытки сохранены"
|
||||
),
|
||||
)
|
||||
status = models.CharField(
|
||||
_("статус"),
|
||||
@@ -356,7 +358,7 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
||||
]
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})"
|
||||
return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m-%d})"
|
||||
|
||||
|
||||
GENERIC_RECORD_SOURCE_CHOICES = [
|
||||
|
||||
@@ -76,7 +76,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
data_scope="Лицензии организаций канонических организаций",
|
||||
task_name="parsers.checko_licenses.refresh",
|
||||
mode="organization_api",
|
||||
access_method="monthly_claim_api",
|
||||
access_method="weekly_claim_api",
|
||||
parser_strategy="scoped_organization_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
@@ -88,7 +88,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
data_scope="Сообщения организаций канонических организаций",
|
||||
task_name="parsers.checko_messages.refresh",
|
||||
mode="organization_api",
|
||||
access_method="monthly_claim_api",
|
||||
access_method="weekly_claim_api",
|
||||
parser_strategy="scoped_organization_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
@@ -100,7 +100,7 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
data_scope="Лизинг канонических организаций",
|
||||
task_name="parsers.checko_leasing.refresh",
|
||||
mode="organization_api",
|
||||
access_method="monthly_claim_api",
|
||||
access_method="weekly_claim_api",
|
||||
parser_strategy="scoped_organization_snapshot",
|
||||
admin_only=True,
|
||||
),
|
||||
|
||||
@@ -21,8 +21,8 @@ from apps.core.services import BackgroundJobService
|
||||
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
||||
from apps.parsers.aggregate_publication import claim_part, finish_part
|
||||
from apps.parsers.checko_collection import (
|
||||
claim_monthly_collection,
|
||||
collection_period_month,
|
||||
claim_weekly_collection,
|
||||
collection_period_week,
|
||||
finish_collection,
|
||||
)
|
||||
from apps.parsers.clients.base import HTTPClientError
|
||||
@@ -209,7 +209,7 @@ def _active_registry_lookup_targets(
|
||||
if checko_source is not None:
|
||||
queryset = queryset.exclude(
|
||||
checko_collection_attempts__source=checko_source,
|
||||
checko_collection_attempts__period_month=collection_period_month(),
|
||||
checko_collection_attempts__period_month=collection_period_week(),
|
||||
)
|
||||
queryset = (
|
||||
queryset.order_by("inn", "ogrn", "uid")
|
||||
@@ -887,7 +887,7 @@ def _fetch_checko_bankruptcy_records(
|
||||
)
|
||||
if not targets:
|
||||
logger.info(
|
||||
"No active registry organizations are due for monthly Fedresurs fallback"
|
||||
"No active registry organizations are due for weekly Fedresurs fallback"
|
||||
)
|
||||
return []
|
||||
|
||||
@@ -897,7 +897,7 @@ def _fetch_checko_bankruptcy_records(
|
||||
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
|
||||
records: list[GenericParserItem] = []
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.BANKRUPTCY,
|
||||
)
|
||||
@@ -1117,7 +1117,7 @@ def _fetch_checko_arbitration_records(
|
||||
subjects = _arbitration_subjects(resolved_limit)
|
||||
if not subjects:
|
||||
raise ParserSourceSkipped(
|
||||
"no active registry organizations are due for monthly arbitration lookup"
|
||||
"no active registry organizations are due for weekly arbitration lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1128,7 +1128,7 @@ def _fetch_checko_arbitration_records(
|
||||
failed_lookups = 0
|
||||
attempted_lookups = 0
|
||||
for subject in subjects:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=subject.registry_organization_id,
|
||||
source=CheckoCollectionAttempt.Source.ARBITRATION,
|
||||
)
|
||||
@@ -1384,7 +1384,7 @@ def _fetch_checko_registry_inspections(
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no active registry organizations are due for monthly inspections lookup"
|
||||
"no active registry organizations are due for weekly inspections lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1395,7 +1395,7 @@ def _fetch_checko_registry_inspections(
|
||||
failed_lookups = 0
|
||||
attempted_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.INSPECTIONS,
|
||||
)
|
||||
@@ -1444,7 +1444,7 @@ def _fetch_eis_registry_procurement_records(
|
||||
proxies: list[str] | None,
|
||||
organization_ids: list[str] | None = None,
|
||||
) -> list[GenericParserItem]:
|
||||
"""Fetch addressed EIS notices for the next monthly OПК registry batch."""
|
||||
"""Fetch addressed EIS notices for the next weekly OПК registry batch."""
|
||||
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
||||
if resolved_limit <= 0:
|
||||
return []
|
||||
@@ -1461,7 +1461,7 @@ def _fetch_eis_registry_procurement_records(
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
f"no OПК registry organizations are due for monthly {law}-FZ lookup"
|
||||
f"no OПК registry organizations are due for weekly {law}-FZ lookup"
|
||||
)
|
||||
|
||||
client = EisRegistryProcurementClient(
|
||||
@@ -1472,7 +1472,7 @@ def _fetch_eis_registry_procurement_records(
|
||||
records: list[GenericParserItem] = []
|
||||
failed_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=claim_source,
|
||||
)
|
||||
@@ -1604,7 +1604,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no OПК registry organizations are due for monthly RNP lookup"
|
||||
"no OПК registry organizations are due for weekly RNP lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1615,7 +1615,7 @@ def _fetch_checko_unfair_supplier_records( # noqa: C901
|
||||
failed_lookups = 0
|
||||
rate_limited = False
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
|
||||
)
|
||||
@@ -1780,7 +1780,7 @@ def _fetch_checko_registry_contract_records(
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no active registry organizations are due for monthly contracts lookup"
|
||||
"no active registry organizations are due for weekly contracts lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
@@ -1791,7 +1791,7 @@ def _fetch_checko_registry_contract_records(
|
||||
failed_lookups = 0
|
||||
attempted_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
attempt = claim_weekly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.CONTRACTS,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user