feat: address QA feedback and limit Checko collection

src/apps/parsers/checko_collection.py

@@ -0,0 +1,56 @@
+"""Quota-safe monthly claims for organization lookups in Checko."""
+
+from datetime import date, datetime
+
+from apps.parsers.models import CheckoCollectionAttempt
+from django.utils import timezone
+
+
+def collection_period_month(at: date | datetime | None = None) -> date:
+    if at is None:
+        local_date = timezone.localdate()
+    elif isinstance(at, datetime):
+        local_date = timezone.localtime(at).date() if timezone.is_aware(at) else at.date()
+    else:
+        local_date = at
+    return local_date.replace(day=1)
+
+
+def claim_monthly_collection(
+    *,
+    organization_id: str,
+    source: str,
+    at: date | datetime | None = None,
+) -> CheckoCollectionAttempt | None:
+    """Atomically claim this month's only allowed Checko collection attempt."""
+    attempt, created = CheckoCollectionAttempt.objects.get_or_create(
+        organization_id=organization_id,
+        source=source,
+        period_month=collection_period_month(at),
+        defaults={"status": CheckoCollectionAttempt.Status.IN_PROGRESS},
+    )
+    return attempt if created else None
+
+
+def finish_collection(
+    attempt: CheckoCollectionAttempt,
+    *,
+    records_count: int,
+    error: Exception | str | None = None,
+) -> None:
+    """Persist the outcome without releasing the monthly claim."""
+    attempt.records_count = max(int(records_count), 0)
+    attempt.error_message = str(error or "")
+    attempt.status = (
+        CheckoCollectionAttempt.Status.FAILED
+        if error is not None
+        else CheckoCollectionAttempt.Status.SUCCESS
+    )
+    attempt.save(
+        update_fields=[
+            "records_count",
+            "error_message",
+            "status",
+            "updated_at",
+        ]
+    )

src/apps/parsers/migrations/0027_checkocollectionattempt.py

@@ -0,0 +1,116 @@
+# Generated by Django 5.2.9 on 2026-07-19
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("organizations", "0007_auto_20260607_1017"),
+        ("parsers", "0026_update_fns_financial_schedule"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="CheckoCollectionAttempt",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created_at",
+                    models.DateTimeField(
+                        auto_now_add=True,
+                        db_index=True,
+                        help_text="Дата и время создания записи",
+                        verbose_name="создано",
+                    ),
+                ),
+                (
+                    "updated_at",
+                    models.DateTimeField(
+                        auto_now=True,
+                        help_text="Дата и время последнего обновления",
+                        verbose_name="обновлено",
+                    ),
+                ),
+                (
+                    "source",
+                    models.CharField(
+                        choices=[
+                            ("arbitration", "Арбитражные дела"),
+                            ("bankruptcy", "Банкротства"),
+                            ("contracts", "Контракты"),
+                            ("inspections", "Проверки"),
+                        ],
+                        max_length=32,
+                        verbose_name="источник",
+                    ),
+                ),
+                (
+                    "period_month",
+                    models.DateField(
+                        help_text="Первый день календарного месяца, за который занята попытка",
+                        verbose_name="месяц сбора",
+                    ),
+                ),
+                (
+                    "status",
+                    models.CharField(
+                        choices=[
+                            ("in_progress", "В процессе"),
+                            ("success", "Успешно"),
+                            ("failed", "Ошибка"),
+                        ],
+                        default="in_progress",
+                        max_length=20,
+                        verbose_name="статус",
+                    ),
+                ),
+                (
+                    "records_count",
+                    models.PositiveIntegerField(
+                        default=0,
+                        verbose_name="количество записей",
+                    ),
+                ),
+                (
+                    "error_message",
+                    models.TextField(blank=True, verbose_name="сообщение об ошибке"),
+                ),
+                (
+                    "organization",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="checko_collection_attempts",
+                        to="organizations.organization",
+                        verbose_name="организация",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "попытка сбора Checko",
+                "verbose_name_plural": "попытки сбора Checko",
+                "db_table": "parsers_checko_collection_attempt",
+                "ordering": ["-period_month", "source", "organization_id"],
+                "indexes": [
+                    models.Index(
+                        fields=["source", "period_month"],
+                        name="parsers_che_source_0ca426_idx",
+                    )
+                ],
+                "constraints": [
+                    models.UniqueConstraint(
+                        fields=("organization", "source", "period_month"),
+                        name="unique_checko_collection_per_org_source_month",
+                    )
+                ],
+            },
+        ),
+    ]

src/apps/parsers/models.py

@@ -117,6 +117,63 @@ class ParserBatchSequence(TimestampMixin, models.Model):
         return f"{self.source}: next batch {self.next_batch_id}"
 
 
+class CheckoCollectionAttempt(TimestampMixin, models.Model):
+    """Monthly Checko collection claim for one organization and source."""
+
+    class Source(models.TextChoices):
+        ARBITRATION = "arbitration", _("Арбитражные дела")
+        BANKRUPTCY = "bankruptcy", _("Банкротства")
+        CONTRACTS = "contracts", _("Контракты")
+        INSPECTIONS = "inspections", _("Проверки")
+
+    class Status(models.TextChoices):
+        IN_PROGRESS = "in_progress", _("В процессе")
+        SUCCESS = "success", _("Успешно")
+        FAILED = "failed", _("Ошибка")
+
+    organization = models.ForeignKey(
+        "organizations.Organization",
+        on_delete=models.CASCADE,
+        related_name="checko_collection_attempts",
+        verbose_name=_("организация"),
+    )
+    source = models.CharField(
+        _("источник"),
+        max_length=32,
+        choices=Source.choices,
+    )
+    period_month = models.DateField(
+        _("месяц сбора"),
+        help_text=_("Первый день календарного месяца, за который занята попытка"),
+    )
+    status = models.CharField(
+        _("статус"),
+        max_length=20,
+        choices=Status.choices,
+        default=Status.IN_PROGRESS,
+    )
+    records_count = models.PositiveIntegerField(_("количество записей"), default=0)
+    error_message = models.TextField(_("сообщение об ошибке"), blank=True)
+
+    class Meta:
+        db_table = "parsers_checko_collection_attempt"
+        verbose_name = _("попытка сбора Checko")
+        verbose_name_plural = _("попытки сбора Checko")
+        ordering = ["-period_month", "source", "organization_id"]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["organization", "source", "period_month"],
+                name="unique_checko_collection_per_org_source_month",
+            ),
+        ]
+        indexes = [
+            models.Index(fields=["source", "period_month"]),
+        ]
+
+    def __str__(self) -> str:
+        return f"{self.organization_id}: {self.source} ({self.period_month:%Y-%m})"
+
+
 GENERIC_RECORD_SOURCE_CHOICES = [
     *ParserLoadLog.Source.choices,
     ("hh", _("Вакансии HeadHunter")),

src/apps/parsers/tasks.py

@@ -18,6 +18,11 @@ from pathlib import Path
 
 from apps.core.services import BackgroundJobService
 from apps.core.tasks import PeriodicTask as CorePeriodicTask
+from apps.parsers.checko_collection import (
+    claim_monthly_collection,
+    collection_period_month,
+    finish_collection,
+)
 from apps.parsers.clients.base import HTTPClientError
 from apps.parsers.clients.checko import (
     CheckoClient,
@@ -43,7 +48,7 @@ from apps.parsers.clients.proverki import ProverkiClient
 from apps.parsers.clients.proverki.schemas import Inspection as ProverkiInspection
 from apps.parsers.clients.vacancies import VacanciesClient
 from apps.parsers.clients.zakupki import ZakupkiClient
-from apps.parsers.models import ParserLoadLog
+from apps.parsers.models import CheckoCollectionAttempt, ParserLoadLog
 from apps.parsers.services import (
     FNSReportOrganizationResolutionSkipped,
     FNSReportService,
@@ -171,11 +176,17 @@ def _resolve_lookup_limit(
 def _active_registry_lookup_targets(
     *,
     limit: int | None = None,
+    checko_source: str | None = None,
 ) -> list[RegistryLookupTarget]:
     """Вернуть организации, которые сейчас состоят хотя бы в одном реестре."""
+    queryset = SourceOrganization.objects.filter(opk_registry_membership=True)
+    if checko_source is not None:
+        queryset = queryset.exclude(
+            checko_collection_attempts__source=checko_source,
+            checko_collection_attempts__period_month=collection_period_month(),
+        )
     queryset = (
-        SourceOrganization.objects.filter(opk_registry_membership=True)
-        .order_by("inn", "ogrn", "uid")
+        queryset.order_by("inn", "ogrn", "uid")
         .values(
             "uid",
             "inn",
@@ -817,9 +828,14 @@ def _fetch_checko_bankruptcy_records(
     if limit <= 0:
         logger.info("Fedresurs Checko fallback is disabled by limit=%s", limit)
         return []
-    targets = _active_registry_lookup_targets(limit=limit)
+    targets = _active_registry_lookup_targets(
+        limit=limit,
+        checko_source=CheckoCollectionAttempt.Source.BANKRUPTCY,
+    )
     if not targets:
-        logger.info("No active registry organizations found for Fedresurs fallback")
+        logger.info(
+            "No active registry organizations are due for monthly Fedresurs fallback"
+        )
         return []
 
     checko_proxies = (
@@ -828,11 +844,19 @@ def _fetch_checko_bankruptcy_records(
     client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
     records: list[GenericParserItem] = []
     for target in targets:
+        attempt = claim_monthly_collection(
+            organization_id=target.organization_id,
+            source=CheckoCollectionAttempt.Source.BANKRUPTCY,
+        )
+        if attempt is None:
+            continue
+        records_before = len(records)
         try:
             response = client.get_company(
                 CompanyRequest(inn=target.inn or None, ogrn=target.ogrn or None)
             )
         except CheckoRateLimitError as exc:
+            finish_collection(attempt, records_count=0, error=exc)
             logger.warning(
                 "Checko bankruptcy fallback stopped: quota/rate limit reached "
                 "(status_code=%s)",
@@ -840,6 +864,7 @@ def _fetch_checko_bankruptcy_records(
             )
             break
         except CheckoError as exc:
+            finish_collection(attempt, records_count=0, error=exc)
             logger.info(
                 "Checko bankruptcy lookup skipped for target=%s: %s",
                 target.inn or target.ogrn,
@@ -847,15 +872,18 @@ def _fetch_checko_bankruptcy_records(
             )
             continue
         company = response.data
-        if company is None:
-            continue
-        records.extend(
-            _checko_bankruptcy_items(
-                company=company,
-                fallback_inn=target.inn,
-                fallback_ogrn=target.ogrn,
-                fallback_name=target.name,
+        if company is not None:
+            records.extend(
+                _checko_bankruptcy_items(
+                    company=company,
+                    fallback_inn=target.inn,
+                    fallback_ogrn=target.ogrn,
+                    fallback_name=target.name,
+                )
             )
+        finish_collection(
+            attempt,
+            records_count=len(records) - records_before,
         )
     logger.info("Fetched %d bankruptcy records through Checko fallback", len(records))
     return records
@@ -990,7 +1018,10 @@ def _add_arbitration_subject(
 def _arbitration_subjects(limit: int) -> list[ArbitrationSubject]:
     """Собрать активные организации из реестров для арбитражного lookup."""
     subjects: dict[str, ArbitrationSubject] = {}
-    for target in _active_registry_lookup_targets(limit=limit):
+    for target in _active_registry_lookup_targets(
+        limit=limit,
+        checko_source=CheckoCollectionAttempt.Source.ARBITRATION,
+    ):
         _add_arbitration_subject(
             subjects,
             inn=target.inn,
@@ -1031,7 +1062,7 @@ def _fetch_checko_arbitration_records(
     subjects = _arbitration_subjects(resolved_limit)
     if not subjects:
         raise ParserSourceSkipped(
-            "no active registry organizations found for arbitration"
+            "no active registry organizations are due for monthly arbitration lookup"
         )
 
     checko_proxies = (
@@ -1040,7 +1071,16 @@ def _fetch_checko_arbitration_records(
     client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
     records: list[GenericParserItem] = []
     failed_lookups = 0
+    attempted_lookups = 0
     for subject in subjects:
+        attempt = claim_monthly_collection(
+            organization_id=subject.registry_organization_id,
+            source=CheckoCollectionAttempt.Source.ARBITRATION,
+        )
+        if attempt is None:
+            continue
+        attempted_lookups += 1
+        records_before = len(records)
         try:
             request = LegalCasesRequest(
                 inn=subject.inn or None,
@@ -1051,13 +1091,19 @@ def _fetch_checko_arbitration_records(
                 records.append(_checko_arbitration_item(legal_case, subject=subject))
         except CheckoError as exc:
             failed_lookups += 1
+            finish_collection(attempt, records_count=0, error=exc)
             logger.info(
                 "Checko arbitration lookup skipped for subject=%s: %s",
                 _arbitration_subject_key(subject),
                 exc,
             )
+        else:
+            finish_collection(
+                attempt,
+                records_count=len(records) - records_before,
+            )
 
-    if failed_lookups == len(subjects) and not records:
+    if attempted_lookups and failed_lookups == attempted_lookups and not records:
         raise ParserSourceSkipped("Checko arbitration lookups failed for all subjects")
 
     logger.info(
@@ -1273,10 +1319,13 @@ def _fetch_checko_registry_inspections(
         logger.info("Registry inspections Checko parser is disabled by limit=%s", limit)
         return []
 
-    targets = _active_registry_lookup_targets(limit=resolved_limit)
+    targets = _active_registry_lookup_targets(
+        limit=resolved_limit,
+        checko_source=CheckoCollectionAttempt.Source.INSPECTIONS,
+    )
     if not targets:
         raise ParserSourceSkipped(
-            "no active registry organizations found for registry inspections"
+            "no active registry organizations are due for monthly inspections lookup"
         )
 
     checko_proxies = (
@@ -1285,7 +1334,16 @@ def _fetch_checko_registry_inspections(
     client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
     records: list[ProverkiInspection] = []
     failed_lookups = 0
+    attempted_lookups = 0
     for target in targets:
+        attempt = claim_monthly_collection(
+            organization_id=target.organization_id,
+            source=CheckoCollectionAttempt.Source.INSPECTIONS,
+        )
+        if attempt is None:
+            continue
+        attempted_lookups += 1
+        records_before = len(records)
         try:
             request = InspectionsRequest(
                 inn=target.inn or None,
@@ -1296,13 +1354,19 @@ def _fetch_checko_registry_inspections(
                 records.append(_checko_inspection_item(inspection, target=target))
         except CheckoError as exc:
             failed_lookups += 1
+            finish_collection(attempt, records_count=0, error=exc)
             logger.info(
                 "Checko inspections lookup skipped for target=%s: %s",
                 target.inn or target.ogrn,
                 exc,
             )
+        else:
+            finish_collection(
+                attempt,
+                records_count=len(records) - records_before,
+            )
 
-    if failed_lookups == len(targets) and not records:
+    if attempted_lookups and failed_lookups == attempted_lookups and not records:
         raise ParserSourceSkipped("Checko inspections lookups failed for all targets")
 
     logger.info(
@@ -1429,10 +1493,13 @@ def _fetch_checko_registry_contract_records(
         logger.info("Registry contracts Checko parser is disabled by limit=%s", limit)
         return []
 
-    targets = _active_registry_lookup_targets(limit=resolved_limit)
+    targets = _active_registry_lookup_targets(
+        limit=resolved_limit,
+        checko_source=CheckoCollectionAttempt.Source.CONTRACTS,
+    )
     if not targets:
         raise ParserSourceSkipped(
-            "no active registry organizations found for registry contracts"
+            "no active registry organizations are due for monthly contracts lookup"
         )
 
     checko_proxies = (
@@ -1441,7 +1508,17 @@ def _fetch_checko_registry_contract_records(
     client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
     records: list[GenericParserItem] = []
     failed_lookups = 0
+    attempted_lookups = 0
     for target in targets:
+        attempt = claim_monthly_collection(
+            organization_id=target.organization_id,
+            source=CheckoCollectionAttempt.Source.CONTRACTS,
+        )
+        if attempt is None:
+            continue
+        attempted_lookups += 1
+        records_before = len(records)
+        target_failures: list[CheckoError] = []
         for law in (ContractLaw.FZ44, ContractLaw.FZ223):
             try:
                 request = ContractsRequest(
@@ -1456,15 +1533,21 @@ def _fetch_checko_registry_contract_records(
                     )
             except CheckoError as exc:
                 failed_lookups += 1
+                target_failures.append(exc)
                 logger.info(
                     "Checko contracts lookup skipped for target=%s law=%s: %s",
                     target.inn or target.ogrn,
                     law.value,
                     exc,
                 )
+        finish_collection(
+            attempt,
+            records_count=len(records) - records_before,
+            error=target_failures[0] if len(target_failures) == 2 else None,
+        )
 
-    expected_lookups = len(targets) * 2
-    if failed_lookups == expected_lookups and not records:
+    expected_lookups = attempted_lookups * 2
+    if expected_lookups and failed_lookups == expected_lookups and not records:
         raise ParserSourceSkipped("Checko contracts lookups failed for all targets")
 
     logger.info(
@@ -2242,6 +2325,9 @@ def parse_all_sources(
                     proxies=proxies
                 ).id,
                 "contracts": parse_contracts.delay(proxies=proxies).id,
+                "registry_contracts": parse_registry_contracts.delay(
+                    proxies=proxies
+                ).id,
                 "unfair_suppliers": parse_unfair_suppliers.delay(proxies=proxies).id,
                 "fas_goz": parse_fas_goz_evasion.delay(proxies=proxies).id,
                 "fns_financial": sync_fns_financial_reports.delay(proxies=proxies).id,
@@ -2249,6 +2335,9 @@ def parse_all_sources(
                 "fedresurs_bankruptcy": parse_fedresurs_bankruptcy.delay(
                     proxies=proxies
                 ).id,
+                "registry_inspections": parse_registry_inspections.delay(
+                    proxies=proxies
+                ).id,
                 "fstec": parse_fstec_registers.delay(proxies=proxies).id,
                 "trudvsem": parse_trudvsem_vacancies.delay(proxies=proxies).id,
             }