fix parser schedule run issues
All checks were successful
CI/CD Pipeline / Manual Action Help (push) Has been skipped
CI/CD Pipeline / Start Dev Containers in Dokploy (push) Has been skipped
CI/CD Pipeline / Drop and Recreate Dev Database (push) Has been skipped
CI/CD Pipeline / Quality Gate (push) Successful in 1m53s
CI/CD Pipeline / Build and Push Images (push) Successful in 2m42s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s

This commit is contained in:
2026-04-28 13:58:55 +02:00
parent b373341fcd
commit c72343a375
7 changed files with 179 additions and 25 deletions

View File

@@ -0,0 +1,18 @@
# Generated by Django 3.2.25 on 2026-04-28 11:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('parsers', '0018_seed_weekly_parser_schedules'),
]
operations = [
migrations.AlterField(
model_name='genericparserrecord',
name='record_date',
field=models.CharField(blank=True, db_index=True, help_text='Дата записи в формате источника', max_length=255, verbose_name='дата записи'),
),
]

View File

@@ -405,7 +405,7 @@ class GenericParserRecord(TimestampMixin, models.Model):
)
record_date = models.CharField(
_("дата записи"),
max_length=30,
max_length=255,
blank=True,
db_index=True,
help_text=_("Дата записи в формате источника"),

View File

@@ -914,6 +914,11 @@ def sync_inspections( # noqa: C901
proxies: list[str] | None = None,
client_adapter: BaseAdapter | None = None,
use_playwright: bool | None = None,
max_months_per_law: int | None = None,
start_year: int | None = None,
start_month: int | None = None,
include_fz294: bool | None = None,
include_fz248: bool | None = None,
current_year: int | None = None,
current_month: int | None = None,
requested_by_id: int | None = None,
@@ -932,6 +937,11 @@ def sync_inspections( # noqa: C901
proxies: Список прокси-серверов (опционально)
client_adapter: HTTP-адаптер (опционально).
use_playwright: Использовать Playwright (опционально).
max_months_per_law: Максимум месяцев для каждого закона.
start_year: Год стартового периода, если нужно переопределить resume.
start_month: Месяц стартового периода, если нужно переопределить resume.
include_fz294: Загружать проверки по ФЗ-294.
include_fz248: Загружать проверки по ФЗ-248.
current_year: Год (опционально) для ограничения периода.
current_month: Месяц (опционально) для ограничения периода.
@@ -967,6 +977,10 @@ def sync_inspections( # noqa: C901
now = datetime.now()
current_year = current_year or now.year
current_month = current_month or now.month
include_fz294 = True if include_fz294 is None else include_fz294
include_fz248 = True if include_fz248 is None else include_fz248
if max_months_per_law is not None:
max_months_per_law = max(1, int(max_months_per_law))
total_saved = 0
results = {"fz294": [], "fz248": []}
@@ -978,43 +992,68 @@ def sync_inspections( # noqa: C901
client_kwargs["use_playwright"] = use_playwright
with ProverkiClient(**client_kwargs) as client:
# Обрабатываем оба типа проверок
for is_fz248 in [False, True]:
law_modes = []
if include_fz294:
law_modes.append(False)
if include_fz248:
law_modes.append(True)
for is_fz248 in law_modes:
fz_key = "fz248" if is_fz248 else "fz294"
fz_name = "ФЗ-248" if is_fz248 else "ФЗ-294"
# Определяем начальную точку
last_year, last_month = InspectionService.get_last_loaded_period(
is_federal_law_248=is_fz248
)
if last_year and last_month:
# Начинаем со следующего месяца после последнего загруженного
start_year, start_month = _get_next_month(last_year, last_month)
if start_year and start_month:
year, month = start_year, start_month
logger.info(
"%s: continuing from %d/%d (last loaded: %d/%d)",
"%s: starting from explicit period %d/%d",
fz_name,
start_year,
start_month,
last_year,
last_month,
year,
month,
)
else:
# Начинаем с дефолтной даты
start_year, start_month = DEFAULT_START_YEAR, DEFAULT_START_MONTH
logger.info(
"%s: no data in DB, starting from %d/%d",
fz_name,
start_year,
start_month,
last_year, last_month = InspectionService.get_last_loaded_period(
is_federal_law_248=is_fz248
)
# Загружаем месяц за месяцем
year, month = start_year, start_month
if last_year and last_month:
# Начинаем со следующего месяца после последнего загруженного
year, month = _get_next_month(last_year, last_month)
logger.info(
"%s: continuing from %d/%d (last loaded: %d/%d)",
fz_name,
year,
month,
last_year,
last_month,
)
else:
# Начинаем с дефолтной даты
year, month = DEFAULT_START_YEAR, DEFAULT_START_MONTH
logger.info(
"%s: no data in DB, starting from %d/%d",
fz_name,
year,
month,
)
empty_months_count = 0
processed_months = 0
while year < current_year or (
year == current_year and month <= current_month
):
if (
max_months_per_law is not None
and processed_months >= max_months_per_law
):
logger.info(
"%s: stopping after %d processed months by request limit",
fz_name,
processed_months,
)
break
# Прекращаем если 2 месяца подряд нет данных
if empty_months_count >= 2:
logger.info(
@@ -1082,7 +1121,8 @@ def sync_inspections( # noqa: C901
)
empty_months_count += 1
# Переходим к следующему месяцу
processed_months += 1
# Переходим к следующему месяцу.
year, month = _get_next_month(year, month)
# Обновляем лог