fix: collect enabled sources weekly including Checko

This commit is contained in:
Aleksandr Meshchryakov
2026-10-29 17:46:26 +01:00
parent aeade84761
commit bc9767244d
12 changed files with 196 additions and 59 deletions

View File

@@ -7,7 +7,7 @@ from unittest.mock import patch
import pytest
from apps.core.models import BackgroundJob
from apps.parsers import checko_section_snapshots as snapshots
from apps.parsers.checko_collection import claim_monthly_collection
from apps.parsers.checko_collection import claim_weekly_collection
from apps.parsers.models import ParserSourceArtifact
from apps.parsers.registry_snapshots import SnapshotValidationError
from apps.parsers.source_cards import SourceCardService
@@ -174,7 +174,7 @@ class CheckoSectionContractTests(TestCase):
def test_no_completed_failure_exposes_rejected_counters(self):
source = "checko_messages"
claim_monthly_collection(organization_id=str(self.org.uid), source=source)
claim_weekly_collection(organization_id=str(self.org.uid), source=source)
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_task(source)
job = BackgroundJob.objects.get()

View File

@@ -1,7 +1,7 @@
from datetime import date
from apps.parsers import tasks as parser_tasks
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.models import CheckoCollectionAttempt
from django.test import TestCase
@@ -18,24 +18,24 @@ class CheckoCollectionClaimTest(TestCase):
mn_okpo="12345678",
)
def test_claim_is_unique_per_organization_source_and_calendar_month(self):
first = claim_monthly_collection(
def test_claim_is_unique_per_organization_source_and_calendar_week(self):
first = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.ARBITRATION,
at=date(2026, 7, 1),
at=date(2026, 7, 25),
)
duplicate = claim_monthly_collection(
duplicate = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.ARBITRATION,
at=date(2026, 7, 31),
)
other_source = claim_monthly_collection(
other_source = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.INSPECTIONS,
at=date(2026, 7, 31),
)
next_month = claim_monthly_collection(
next_month = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.ARBITRATION,
at=date(2026, 8, 1),
@@ -46,8 +46,8 @@ class CheckoCollectionClaimTest(TestCase):
self.assertIsNotNone(other_source)
self.assertIsNotNone(next_month)
def test_failed_attempt_keeps_monthly_claim(self):
attempt = claim_monthly_collection(
def test_failed_attempt_keeps_weekly_claim(self):
attempt = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.CONTRACTS,
at=date(2026, 7, 19),
@@ -55,7 +55,7 @@ class CheckoCollectionClaimTest(TestCase):
assert attempt is not None
finish_collection(attempt, records_count=0, error="quota exhausted")
repeated = claim_monthly_collection(
repeated = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.CONTRACTS,
at=date(2026, 7, 20),
@@ -78,7 +78,7 @@ class CheckoCollectionClaimTest(TestCase):
)
second.opk_registry_membership = True
second.save(update_fields=["opk_registry_membership"])
claimed = claim_monthly_collection(
claimed = claim_weekly_collection(
organization_id=str(self.organization.id),
source=CheckoCollectionAttempt.Source.INSPECTIONS,
)
@@ -115,3 +115,15 @@ class CheckoCollectionClaimTest(TestCase):
[target.organization_id for target in targets],
[str(self.organization.id)],
)
def test_week_boundary_uses_moscow_saturday():
from datetime import datetime, timezone
from apps.parsers.checko_collection import collection_period_week
assert collection_period_week(
datetime(2026, 10, 30, 20, 59, tzinfo=timezone.utc)
) == date(2026, 10, 24)
assert collection_period_week(
datetime(2026, 10, 30, 21, 0, tzinfo=timezone.utc)
) == date(2026, 10, 31)

View File

@@ -6,7 +6,7 @@ from tempfile import TemporaryDirectory
from unittest.mock import Mock, patch
import pytest
from apps.parsers.checko_collection import claim_monthly_collection, finish_collection
from apps.parsers.checko_collection import claim_weekly_collection, finish_collection
from apps.parsers.models import (
CheckoCollectionAttempt,
ParserSourceArtifact,
@@ -96,7 +96,7 @@ class CheckoSectionSnapshotTests(TestCase):
def test_claimed_organization_keeps_previous_records(self):
self.seed(self.orgs[0])
before = self.existing(self.orgs[0])
claim_monthly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
claim_weekly_collection(organization_id=str(self.orgs[0].uid), source=SOURCE)
(artifact, result), collector = self.run_refresh(rows=[[record(self.orgs[1])]])
assert self.existing(self.orgs[0]) == before
assert collector.call_count == 1
@@ -134,7 +134,7 @@ class CheckoSectionSnapshotTests(TestCase):
for org in self.orgs:
self.seed(org)
before = self.existing(self.orgs[1])
claim_monthly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
claim_weekly_collection(organization_id=str(self.orgs[1].uid), source=SOURCE)
self.run_refresh(rows=[[]])
assert not self.existing(self.orgs[0])
assert self.existing(self.orgs[1]) == before
@@ -163,7 +163,7 @@ class CheckoSectionSnapshotTests(TestCase):
)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(
claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=SOURCE
)
is None
@@ -172,7 +172,7 @@ class CheckoSectionSnapshotTests(TestCase):
def test_no_completed_scope_rejects_without_publication(self):
for org in self.orgs:
self.seed(org)
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
before = [self.existing(org) for org in self.orgs]
with pytest.raises(SnapshotValidationError, match="no_completed_organizations"):
self.run_refresh()
@@ -220,7 +220,7 @@ class CheckoSectionSnapshotTests(TestCase):
attempt = CheckoCollectionAttempt.objects.get(organization=org, source=SOURCE)
assert attempt.status == attempt.Status.FAILED
assert (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
is None
)
@@ -246,7 +246,7 @@ class CheckoSectionSnapshotTests(TestCase):
before = [self.existing(org)[0][0] for org in self.orgs]
# Advance the calendar; never delete or reset a monthly claim.
with patch(
"apps.parsers.checko_collection.collection_period_month",
"apps.parsers.checko_collection.collection_period_week",
return_value=date(2026, 11, 1),
):
self.run_refresh(load_batch=2)
@@ -257,13 +257,13 @@ class CheckoSectionSnapshotTests(TestCase):
CheckoCollectionAttempt.Source.values
)
for source in adapter().CHECKO_SECTIONS:
attempt = claim_monthly_collection(
attempt = claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
assert attempt is not None
finish_collection(attempt, records_count=0, error="fixture")
assert (
claim_monthly_collection(
claim_weekly_collection(
organization_id=str(self.orgs[0].uid), source=source
)
is None
@@ -311,7 +311,7 @@ def test_concurrent_monthly_claim_one_winner():
def claim():
return (
claim_monthly_collection(organization_id=str(org.uid), source=SOURCE)
claim_weekly_collection(organization_id=str(org.uid), source=SOURCE)
is not None
)

View File

@@ -161,3 +161,41 @@ class RequestedSourceScheduleTest(TestCase):
migration.disable_schedules(apps, None)
assert not PeriodicTask.objects.filter(name__in=expected, enabled=True).exists()
assert PeriodicTask.objects.filter(name__in=expected).count() == 5
class WeeklySourcePolicyTest(TestCase):
def test_policy_consolidates_enabled_sources_and_preserves_disabled_gates(self):
migration = importlib.import_module(
"apps.parsers.migrations.0050_weekly_source_refresh"
)
interval = IntervalSchedule.objects.create(every=30, period="days")
source, _ = PeriodicTask.objects.update_or_create(
name="parser:checko-leasing:monthly",
defaults={
"task": "parsers.checko_leasing.refresh",
"enabled": True,
"interval": interval,
"crontab": None,
},
)
gated, _ = PeriodicTask.objects.update_or_create(
name="parser:sro-membership:full-msk",
defaults={
"task": "parsers.sro_membership_check.refresh",
"enabled": False,
"interval": interval,
},
)
migration.weekly_sources(apps, None)
migration.weekly_sources(apps, None)
source.refresh_from_db()
gated.refresh_from_db()
self.assertIsNone(source.interval_id)
self.assertEqual(source.crontab.day_of_week, "6")
self.assertEqual(str(source.crontab.timezone), "Europe/Moscow")
self.assertFalse(gated.enabled)
self.assertFalse(
PeriodicTask.objects.filter(
name__in=migration.DUPLICATES, enabled=True
).exists()
)

View File

@@ -324,7 +324,7 @@ class CheckoFedresursFallbackControlFlowTestCase(SimpleTestCase):
patch.object(parser_tasks, "CheckoClient", _RateLimitedCheckoClient),
patch.object(
parser_tasks,
"claim_monthly_collection",
"claim_weekly_collection",
return_value=SimpleNamespace(),
),
patch.object(parser_tasks, "finish_collection"),