|
|
|
|
@@ -32,6 +32,7 @@ from apps.parsers.services import (
|
|
|
|
|
ProxyService,
|
|
|
|
|
)
|
|
|
|
|
from celery import shared_task
|
|
|
|
|
from django.conf import settings
|
|
|
|
|
from requests.adapters import BaseAdapter
|
|
|
|
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
@@ -41,6 +42,26 @@ DEFAULT_START_YEAR = 2025
|
|
|
|
|
DEFAULT_START_MONTH = 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _resolve_proxies(proxies: list[str] | None) -> list[str] | None:
|
|
|
|
|
"""
|
|
|
|
|
Разрешить итоговый список прокси.
|
|
|
|
|
|
|
|
|
|
Приоритет:
|
|
|
|
|
1. Явно переданные в задачу `proxies`
|
|
|
|
|
2. Активные прокси из БД
|
|
|
|
|
3. `settings.PARSER_PROXIES` (например, из ENV)
|
|
|
|
|
"""
|
|
|
|
|
if proxies is not None:
|
|
|
|
|
return proxies
|
|
|
|
|
|
|
|
|
|
db_proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
if db_proxies:
|
|
|
|
|
return db_proxies
|
|
|
|
|
|
|
|
|
|
configured_proxies = getattr(settings, "PARSER_PROXIES", []) or []
|
|
|
|
|
return configured_proxies or None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _get_or_create_background_job(
|
|
|
|
|
*,
|
|
|
|
|
task_id: str,
|
|
|
|
|
@@ -301,9 +322,7 @@ def parse_industrial_production(
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
# Если прокси не переданы, берём из БД
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting industrial production parsing (task_id=%s, batch_id=%d, proxies=%d)",
|
|
|
|
|
@@ -395,9 +414,7 @@ def parse_manufactures(
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
# Если прокси не переданы, берём из БД
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting manufactures parsing (task_id=%s, batch_id=%d, proxies=%d)",
|
|
|
|
|
@@ -488,8 +505,7 @@ def parse_industrial_products(
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting industrial products parsing (task_id=%s, batch_id=%d, proxies=%d)",
|
|
|
|
|
@@ -634,9 +650,7 @@ def parse_inspections(
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
# Если прокси не переданы, берём из БД
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting inspections parsing (task_id=%s, batch_id=%d, year=%s, month=%s, proxies=%d)",
|
|
|
|
|
@@ -828,9 +842,7 @@ def sync_inspections( # noqa: C901
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
# Если прокси не переданы, берём из БД
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting inspections sync (task_id=%s, batch_id=%d)", task_id, batch_id
|
|
|
|
|
@@ -1039,9 +1051,7 @@ def parse_procurements(
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
# Если прокси не переданы, берём из БД
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting procurements parsing "
|
|
|
|
|
@@ -1174,9 +1184,7 @@ def sync_procurements( # noqa: C901
|
|
|
|
|
)
|
|
|
|
|
task_id = self.request.id or str(uuid.uuid4())
|
|
|
|
|
|
|
|
|
|
# Если прокси не переданы, берём из БД
|
|
|
|
|
if proxies is None:
|
|
|
|
|
proxies = ProxyService.get_active_proxies_or_none()
|
|
|
|
|
proxies = _resolve_proxies(proxies)
|
|
|
|
|
|
|
|
|
|
logger.info(
|
|
|
|
|
"Starting procurements sync (task_id=%s, batch_id=%d, region=%s, law=%s-FZ)",
|
|
|
|
|
|