fix parser schedule run issues
All checks were successful
CI/CD Pipeline / Manual Action Help (push) Has been skipped
CI/CD Pipeline / Start Dev Containers in Dokploy (push) Has been skipped
CI/CD Pipeline / Drop and Recreate Dev Database (push) Has been skipped
CI/CD Pipeline / Quality Gate (push) Successful in 1m53s
CI/CD Pipeline / Build and Push Images (push) Successful in 2m42s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s
All checks were successful
CI/CD Pipeline / Manual Action Help (push) Has been skipped
CI/CD Pipeline / Start Dev Containers in Dokploy (push) Has been skipped
CI/CD Pipeline / Drop and Recreate Dev Database (push) Has been skipped
CI/CD Pipeline / Quality Gate (push) Successful in 1m53s
CI/CD Pipeline / Build and Push Images (push) Successful in 2m42s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
# Generated by Django 3.2.25 on 2026-04-28 11:52
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('parsers', '0018_seed_weekly_parser_schedules'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='genericparserrecord',
|
||||
name='record_date',
|
||||
field=models.CharField(blank=True, db_index=True, help_text='Дата записи в формате источника', max_length=255, verbose_name='дата записи'),
|
||||
),
|
||||
]
|
||||
@@ -405,7 +405,7 @@ class GenericParserRecord(TimestampMixin, models.Model):
|
||||
)
|
||||
record_date = models.CharField(
|
||||
_("дата записи"),
|
||||
max_length=30,
|
||||
max_length=255,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
help_text=_("Дата записи в формате источника"),
|
||||
|
||||
@@ -914,6 +914,11 @@ def sync_inspections( # noqa: C901
|
||||
proxies: list[str] | None = None,
|
||||
client_adapter: BaseAdapter | None = None,
|
||||
use_playwright: bool | None = None,
|
||||
max_months_per_law: int | None = None,
|
||||
start_year: int | None = None,
|
||||
start_month: int | None = None,
|
||||
include_fz294: bool | None = None,
|
||||
include_fz248: bool | None = None,
|
||||
current_year: int | None = None,
|
||||
current_month: int | None = None,
|
||||
requested_by_id: int | None = None,
|
||||
@@ -932,6 +937,11 @@ def sync_inspections( # noqa: C901
|
||||
proxies: Список прокси-серверов (опционально)
|
||||
client_adapter: HTTP-адаптер (опционально).
|
||||
use_playwright: Использовать Playwright (опционально).
|
||||
max_months_per_law: Максимум месяцев для каждого закона.
|
||||
start_year: Год стартового периода, если нужно переопределить resume.
|
||||
start_month: Месяц стартового периода, если нужно переопределить resume.
|
||||
include_fz294: Загружать проверки по ФЗ-294.
|
||||
include_fz248: Загружать проверки по ФЗ-248.
|
||||
current_year: Год (опционально) для ограничения периода.
|
||||
current_month: Месяц (опционально) для ограничения периода.
|
||||
|
||||
@@ -967,6 +977,10 @@ def sync_inspections( # noqa: C901
|
||||
now = datetime.now()
|
||||
current_year = current_year or now.year
|
||||
current_month = current_month or now.month
|
||||
include_fz294 = True if include_fz294 is None else include_fz294
|
||||
include_fz248 = True if include_fz248 is None else include_fz248
|
||||
if max_months_per_law is not None:
|
||||
max_months_per_law = max(1, int(max_months_per_law))
|
||||
total_saved = 0
|
||||
results = {"fz294": [], "fz248": []}
|
||||
|
||||
@@ -978,43 +992,68 @@ def sync_inspections( # noqa: C901
|
||||
client_kwargs["use_playwright"] = use_playwright
|
||||
with ProverkiClient(**client_kwargs) as client:
|
||||
# Обрабатываем оба типа проверок
|
||||
for is_fz248 in [False, True]:
|
||||
law_modes = []
|
||||
if include_fz294:
|
||||
law_modes.append(False)
|
||||
if include_fz248:
|
||||
law_modes.append(True)
|
||||
|
||||
for is_fz248 in law_modes:
|
||||
fz_key = "fz248" if is_fz248 else "fz294"
|
||||
fz_name = "ФЗ-248" if is_fz248 else "ФЗ-294"
|
||||
|
||||
# Определяем начальную точку
|
||||
last_year, last_month = InspectionService.get_last_loaded_period(
|
||||
is_federal_law_248=is_fz248
|
||||
)
|
||||
|
||||
if last_year and last_month:
|
||||
# Начинаем со следующего месяца после последнего загруженного
|
||||
start_year, start_month = _get_next_month(last_year, last_month)
|
||||
if start_year and start_month:
|
||||
year, month = start_year, start_month
|
||||
logger.info(
|
||||
"%s: continuing from %d/%d (last loaded: %d/%d)",
|
||||
"%s: starting from explicit period %d/%d",
|
||||
fz_name,
|
||||
start_year,
|
||||
start_month,
|
||||
last_year,
|
||||
last_month,
|
||||
year,
|
||||
month,
|
||||
)
|
||||
else:
|
||||
# Начинаем с дефолтной даты
|
||||
start_year, start_month = DEFAULT_START_YEAR, DEFAULT_START_MONTH
|
||||
logger.info(
|
||||
"%s: no data in DB, starting from %d/%d",
|
||||
fz_name,
|
||||
start_year,
|
||||
start_month,
|
||||
last_year, last_month = InspectionService.get_last_loaded_period(
|
||||
is_federal_law_248=is_fz248
|
||||
)
|
||||
|
||||
# Загружаем месяц за месяцем
|
||||
year, month = start_year, start_month
|
||||
if last_year and last_month:
|
||||
# Начинаем со следующего месяца после последнего загруженного
|
||||
year, month = _get_next_month(last_year, last_month)
|
||||
logger.info(
|
||||
"%s: continuing from %d/%d (last loaded: %d/%d)",
|
||||
fz_name,
|
||||
year,
|
||||
month,
|
||||
last_year,
|
||||
last_month,
|
||||
)
|
||||
else:
|
||||
# Начинаем с дефолтной даты
|
||||
year, month = DEFAULT_START_YEAR, DEFAULT_START_MONTH
|
||||
logger.info(
|
||||
"%s: no data in DB, starting from %d/%d",
|
||||
fz_name,
|
||||
year,
|
||||
month,
|
||||
)
|
||||
|
||||
empty_months_count = 0
|
||||
processed_months = 0
|
||||
|
||||
while year < current_year or (
|
||||
year == current_year and month <= current_month
|
||||
):
|
||||
if (
|
||||
max_months_per_law is not None
|
||||
and processed_months >= max_months_per_law
|
||||
):
|
||||
logger.info(
|
||||
"%s: stopping after %d processed months by request limit",
|
||||
fz_name,
|
||||
processed_months,
|
||||
)
|
||||
break
|
||||
|
||||
# Прекращаем если 2 месяца подряд нет данных
|
||||
if empty_months_count >= 2:
|
||||
logger.info(
|
||||
@@ -1082,7 +1121,8 @@ def sync_inspections( # noqa: C901
|
||||
)
|
||||
empty_months_count += 1
|
||||
|
||||
# Переходим к следующему месяцу
|
||||
processed_months += 1
|
||||
# Переходим к следующему месяцу.
|
||||
year, month = _get_next_month(year, month)
|
||||
|
||||
# Обновляем лог
|
||||
|
||||
Reference in New Issue
Block a user