feat: restore registry procurement collection
This commit is contained in:
@@ -193,7 +193,18 @@ RU_FIELD_MAP = {
|
||||
"ЕФРСБ": "bankruptcy",
|
||||
"НомерДела": "case_number",
|
||||
# РНП
|
||||
"НедобПост": "unfair_supplier",
|
||||
"НедобПост": "is_unfair_supplier",
|
||||
"НедобПостЗап": "unfair_supplier",
|
||||
"РеестрНомер": "registry_number",
|
||||
"ДатаПуб": "publish_date",
|
||||
"ДатаУтв": "approval_date",
|
||||
"ЗаказНаимСокр": "customer_short_name",
|
||||
"ЗаказНаимПолн": "customer_full_name",
|
||||
"ЗаказИНН": "customer_inn",
|
||||
"ЗаказКПП": "customer_kpp",
|
||||
"ЗакупНомер": "purchase_number",
|
||||
"ЗакупОпис": "purchase_description",
|
||||
"ЦенаКонтр": "contract_price",
|
||||
# Численность
|
||||
"СЧР": "employees_count",
|
||||
# Налоги
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Общие клиенты и DTO для новых разнородных источников."""
|
||||
|
||||
from apps.parsers.clients.common.eis_registry import EisRegistryProcurementClient
|
||||
from apps.parsers.clients.common.schemas import GenericParserItem
|
||||
from apps.parsers.clients.common.structured import (
|
||||
StructuredDataClient,
|
||||
@@ -8,6 +9,7 @@ from apps.parsers.clients.common.structured import (
|
||||
|
||||
__all__ = [
|
||||
"GenericParserItem",
|
||||
"EisRegistryProcurementClient",
|
||||
"StructuredDataClient",
|
||||
"StructuredDataClientError",
|
||||
]
|
||||
|
||||
142
src/apps/parsers/clients/common/eis_registry.py
Normal file
142
src/apps/parsers/clients/common/eis_registry.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Addressed EIS procurement lookup for organizations from the OПК registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import date
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from apps.parsers.clients.common.schemas import GenericParserItem
|
||||
from apps.parsers.clients.common.structured import StructuredDataClient
|
||||
from bs4 import BeautifulSoup
|
||||
from dateutil.relativedelta import relativedelta
|
||||
|
||||
EIS_PROCUREMENT_SEARCH_URL = (
|
||||
"https://zakupki.gov.ru/epz/order/extendedsearch/results.html"
|
||||
)
|
||||
EIS_SEARCH_PAGE_MAX_SIZE_BYTES = 4 * 1024 * 1024
|
||||
EIS_RECORDS_PER_PAGE = 50
|
||||
|
||||
|
||||
def _digits(value: str) -> str:
|
||||
return re.sub(r"\D+", "", str(value or ""))
|
||||
|
||||
|
||||
@dataclass
|
||||
class EisRegistryProcurementClient:
|
||||
"""Fetch all recent EIS notices for one exact customer INN and law."""
|
||||
|
||||
source: str
|
||||
law: str
|
||||
proxies: list[str] | None = None
|
||||
timeout: int = 120
|
||||
lookback_months: int = 12
|
||||
max_pages: int = 1000
|
||||
_structured_client: StructuredDataClient | None = field(
|
||||
default=None,
|
||||
repr=False,
|
||||
)
|
||||
|
||||
@property
|
||||
def structured_client(self) -> StructuredDataClient:
|
||||
if self._structured_client is None:
|
||||
self._structured_client = StructuredDataClient(
|
||||
source=self.source,
|
||||
proxies=self.proxies,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
return self._structured_client
|
||||
|
||||
def fetch_for_customer(
|
||||
self,
|
||||
*,
|
||||
inn: str,
|
||||
organization_name: str,
|
||||
organization_id: str,
|
||||
today: date | None = None,
|
||||
) -> list[GenericParserItem]:
|
||||
"""Fetch every result page and keep only cards with the exact customer INN."""
|
||||
customer_inn = _digits(inn)
|
||||
if not customer_inn:
|
||||
return []
|
||||
|
||||
period_end = today or date.today()
|
||||
period_start = period_end - relativedelta(months=self.lookback_months)
|
||||
records_by_external_id: dict[str, GenericParserItem] = {}
|
||||
page_number = 1
|
||||
|
||||
while page_number <= self.max_pages:
|
||||
url = self._build_search_url(
|
||||
inn=customer_inn,
|
||||
page_number=page_number,
|
||||
period_start=period_start,
|
||||
period_end=period_end,
|
||||
)
|
||||
content = self.structured_client.http_client.download_file(
|
||||
url,
|
||||
max_size_bytes=EIS_SEARCH_PAGE_MAX_SIZE_BYTES,
|
||||
)
|
||||
page_records = self.structured_client.fetch_records(
|
||||
content=content,
|
||||
file_name="results.html",
|
||||
)
|
||||
for record in page_records:
|
||||
if _digits(record.inn) != customer_inn:
|
||||
continue
|
||||
payload = dict(record.payload)
|
||||
payload.update(
|
||||
{
|
||||
"provider": "eis",
|
||||
"organization_id": organization_id,
|
||||
"customer_inn": customer_inn,
|
||||
"customer_name": organization_name,
|
||||
"law": self.law,
|
||||
"lookback_months": self.lookback_months,
|
||||
}
|
||||
)
|
||||
records_by_external_id[record.external_id] = replace(
|
||||
record,
|
||||
inn=customer_inn,
|
||||
organisation_name=organization_name,
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
last_page = self._last_page_number(content)
|
||||
if page_number >= last_page:
|
||||
break
|
||||
page_number += 1
|
||||
|
||||
return list(records_by_external_id.values())
|
||||
|
||||
def _build_search_url(
|
||||
self,
|
||||
*,
|
||||
inn: str,
|
||||
page_number: int,
|
||||
period_start: date,
|
||||
period_end: date,
|
||||
) -> str:
|
||||
law_param = "fz44" if self.law == "44" else "fz223"
|
||||
params = {
|
||||
law_param: "on",
|
||||
"searchString": inn,
|
||||
"strictEqual": "true",
|
||||
"publishDateFrom": period_start.strftime("%d.%m.%Y"),
|
||||
"publishDateTo": period_end.strftime("%d.%m.%Y"),
|
||||
"sortBy": "UPDATE_DATE",
|
||||
"pageNumber": str(page_number),
|
||||
"sortDirection": "false",
|
||||
"recordsPerPage": f"_{EIS_RECORDS_PER_PAGE}",
|
||||
}
|
||||
return f"{EIS_PROCUREMENT_SEARCH_URL}?{urlencode(params)}"
|
||||
|
||||
@staticmethod
|
||||
def _last_page_number(content: bytes) -> int:
|
||||
soup = BeautifulSoup(content, "html.parser")
|
||||
page_numbers = [1]
|
||||
for node in soup.select("[data-pagenumber]"):
|
||||
value = str(node.get("data-pagenumber") or "")
|
||||
if value.isdigit():
|
||||
page_numbers.append(int(value))
|
||||
return max(page_numbers)
|
||||
@@ -0,0 +1,43 @@
|
||||
from django.db import migrations, models
|
||||
|
||||
LEGACY_WEEKLY_TASK_NAMES = [
|
||||
"parser:procurements_44fz:weekly-saturday-msk",
|
||||
"parser:procurements_223fz:weekly-saturday-msk",
|
||||
"parser:contracts:weekly-saturday-msk",
|
||||
"parser:unfair_suppliers:weekly-saturday-msk",
|
||||
]
|
||||
|
||||
|
||||
def disable_legacy_weekly_tasks(apps, schema_editor):
|
||||
PeriodicTask = apps.get_model("django_celery_beat", "PeriodicTask")
|
||||
PeriodicTask.objects.filter(name__in=LEGACY_WEEKLY_TASK_NAMES).update(enabled=False)
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
dependencies = [
|
||||
("parsers", "0027_checkocollectionattempt"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="checkocollectionattempt",
|
||||
name="source",
|
||||
field=models.CharField(
|
||||
choices=[
|
||||
("arbitration", "Арбитражные дела"),
|
||||
("bankruptcy", "Банкротства"),
|
||||
("contracts", "Контракты"),
|
||||
("inspections", "Проверки"),
|
||||
("procurements_44fz", "Закупки 44-ФЗ"),
|
||||
("procurements_223fz", "Закупки 223-ФЗ"),
|
||||
("unfair_suppliers", "Недобросовестные поставщики"),
|
||||
],
|
||||
max_length=32,
|
||||
verbose_name="источник",
|
||||
),
|
||||
),
|
||||
migrations.RunPython(
|
||||
disable_legacy_weekly_tasks,
|
||||
reverse_code=migrations.RunPython.noop,
|
||||
),
|
||||
]
|
||||
@@ -125,6 +125,9 @@ class CheckoCollectionAttempt(TimestampMixin, models.Model):
|
||||
BANKRUPTCY = "bankruptcy", _("Банкротства")
|
||||
CONTRACTS = "contracts", _("Контракты")
|
||||
INSPECTIONS = "inspections", _("Проверки")
|
||||
PROCUREMENTS_44FZ = "procurements_44fz", _("Закупки 44-ФЗ")
|
||||
PROCUREMENTS_223FZ = "procurements_223fz", _("Закупки 223-ФЗ")
|
||||
UNFAIR_SUPPLIERS = "unfair_suppliers", _("Недобросовестные поставщики")
|
||||
|
||||
class Status(models.TextChoices):
|
||||
IN_PROGRESS = "in_progress", _("В процессе")
|
||||
|
||||
@@ -130,18 +130,16 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
description="Данные ЕИС закупок по тендерам и заказчикам.",
|
||||
order=20,
|
||||
task_names=(
|
||||
"apps.parsers.tasks.parse_procurements",
|
||||
"apps.parsers.tasks.sync_procurements",
|
||||
"apps.parsers.tasks.parse_procurements_44fz",
|
||||
"apps.parsers.tasks.parse_procurements_223fz",
|
||||
"apps.parsers.tasks.parse_contracts",
|
||||
"apps.parsers.tasks.parse_registry_contracts",
|
||||
"apps.parsers.tasks.parse_registry_enrichment_sources",
|
||||
),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="procurements",
|
||||
title="Единая информационная система закупок",
|
||||
description=("Закупки и связанные данные из ЕИС по 44-ФЗ и 223-ФЗ."),
|
||||
title="Общие закупки",
|
||||
description="Объединение закупок 44-ФЗ и 223-ФЗ без дублирования.",
|
||||
parser_source=ParserLoadLog.Source.PROCUREMENTS,
|
||||
),
|
||||
SourceItemDefinition(
|
||||
@@ -168,7 +166,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
name="region_code",
|
||||
label="Код региона",
|
||||
description="Код региона ЕИС, например 77 для Москвы.",
|
||||
required=True,
|
||||
required=False,
|
||||
),
|
||||
RefreshParamDefinition(
|
||||
name="law_type",
|
||||
@@ -368,6 +366,10 @@ SOURCE_RECORD_SOURCES_BY_ITEM_CODE = {
|
||||
for item in definition.source_items
|
||||
if item.parser_source
|
||||
}
|
||||
SOURCE_RECORD_SOURCES_BY_ITEM_CODE["procurements"] = [
|
||||
ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
ParserLoadLog.Source.PROCUREMENTS_223FZ,
|
||||
]
|
||||
|
||||
|
||||
class SourceCardService:
|
||||
@@ -455,7 +457,14 @@ class SourceCardService:
|
||||
source_items = [
|
||||
cls._build_source_item(item, context) for item in definition.source_items
|
||||
]
|
||||
records_count = sum(item["records_count"] for item in source_items)
|
||||
if definition.slug == "public-procurements":
|
||||
items_by_code = {item["code"]: item for item in source_items}
|
||||
records_count = (
|
||||
items_by_code["procurements"]["records_count"]
|
||||
+ items_by_code["contracts"]["records_count"]
|
||||
)
|
||||
else:
|
||||
records_count = sum(item["records_count"] for item in source_items)
|
||||
organizations_count = cls._get_card_organizations_count(
|
||||
definition, source_items, context
|
||||
)
|
||||
@@ -998,27 +1007,18 @@ class SourceCardService:
|
||||
return [task_info]
|
||||
|
||||
if definition.slug == "public-procurements":
|
||||
from apps.parsers.tasks import sync_procurements
|
||||
from apps.parsers.tasks import parse_registry_enrichment_sources
|
||||
|
||||
task_info = cls._enqueue_task(
|
||||
task=sync_procurements,
|
||||
task_name="apps.parsers.tasks.sync_procurements",
|
||||
task=parse_registry_enrichment_sources,
|
||||
task_name="apps.parsers.tasks.parse_registry_enrichment_sources",
|
||||
requested_by_id=requested_by_id,
|
||||
meta={
|
||||
"source_card": definition.slug,
|
||||
"source": ParserLoadLog.Source.PROCUREMENTS,
|
||||
"region_code": params["region_code"],
|
||||
"law_type": params.get("law_type", "44"),
|
||||
"source": ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
},
|
||||
kwargs={
|
||||
"requested_by_id": requested_by_id,
|
||||
"region_code": params["region_code"],
|
||||
"law_type": params.get("law_type", "44"),
|
||||
**{
|
||||
key: value
|
||||
for key, value in params.items()
|
||||
if key in {"current_year", "current_month"}
|
||||
},
|
||||
},
|
||||
)
|
||||
return [task_info]
|
||||
|
||||
@@ -37,7 +37,11 @@ from apps.parsers.clients.checko import (
|
||||
SearchType,
|
||||
)
|
||||
from apps.parsers.clients.checko.exceptions import CheckoError
|
||||
from apps.parsers.clients.common import GenericParserItem, StructuredDataClient
|
||||
from apps.parsers.clients.common import (
|
||||
EisRegistryProcurementClient,
|
||||
GenericParserItem,
|
||||
StructuredDataClient,
|
||||
)
|
||||
from apps.parsers.clients.fns import FNSApiClient, FNSApiReport
|
||||
from apps.parsers.clients.gisp import GispProductsClient
|
||||
from apps.parsers.clients.minpromtorg import (
|
||||
@@ -82,6 +86,7 @@ FEDRESURS_CHECKO_FALLBACK_LIMIT = 100
|
||||
ARBITRATION_CHECKO_LIMIT = 100
|
||||
REGISTRY_INSPECTIONS_CHECKO_LIMIT = 1000
|
||||
REGISTRY_CONTRACTS_CHECKO_LIMIT = 1000
|
||||
REGISTRY_ENRICHMENT_BATCH_SIZE = 250
|
||||
FSTEC_CHECKO_IDENTITY_LOOKUP_LIMIT = 1000
|
||||
PARSER_STALE_LOAD_MAX_AGE_MINUTES = 90
|
||||
PARSER_SOFT_TIME_LIMIT_SECONDS = 15 * 60
|
||||
@@ -177,9 +182,15 @@ def _active_registry_lookup_targets(
|
||||
*,
|
||||
limit: int | None = None,
|
||||
checko_source: str | None = None,
|
||||
require_inn: bool = False,
|
||||
organization_ids: list[str] | None = None,
|
||||
) -> list[RegistryLookupTarget]:
|
||||
"""Вернуть организации, которые сейчас состоят хотя бы в одном реестре."""
|
||||
queryset = SourceOrganization.objects.filter(opk_registry_membership=True)
|
||||
if organization_ids is not None:
|
||||
queryset = queryset.filter(uid__in=organization_ids)
|
||||
if require_inn:
|
||||
queryset = queryset.exclude(inn="")
|
||||
if checko_source is not None:
|
||||
queryset = queryset.exclude(
|
||||
checko_collection_attempts__source=checko_source,
|
||||
@@ -215,6 +226,17 @@ def _active_registry_lookup_targets(
|
||||
return targets
|
||||
|
||||
|
||||
def _resolve_registry_enrichment_limit(limit: int | None) -> int:
|
||||
return _resolve_lookup_limit(
|
||||
limit,
|
||||
default=getattr(
|
||||
settings,
|
||||
"REGISTRY_ENRICHMENT_BATCH_SIZE",
|
||||
REGISTRY_ENRICHMENT_BATCH_SIZE,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_proxies(proxies: list[str] | None) -> list[str] | None:
|
||||
"""
|
||||
Разрешить итоговый список прокси.
|
||||
@@ -1377,6 +1399,226 @@ def _fetch_checko_registry_inspections(
|
||||
return records
|
||||
|
||||
|
||||
def _fetch_eis_registry_procurement_records(
|
||||
*,
|
||||
source: str,
|
||||
law: str,
|
||||
limit: int | None,
|
||||
proxies: list[str] | None,
|
||||
organization_ids: list[str] | None = None,
|
||||
) -> list[GenericParserItem]:
|
||||
"""Fetch addressed EIS notices for the next monthly OПК registry batch."""
|
||||
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
||||
if resolved_limit <= 0:
|
||||
return []
|
||||
|
||||
claim_source = {
|
||||
"44": CheckoCollectionAttempt.Source.PROCUREMENTS_44FZ,
|
||||
"223": CheckoCollectionAttempt.Source.PROCUREMENTS_223FZ,
|
||||
}[law]
|
||||
targets = _active_registry_lookup_targets(
|
||||
limit=resolved_limit,
|
||||
checko_source=claim_source,
|
||||
require_inn=True,
|
||||
organization_ids=organization_ids,
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
f"no OПК registry organizations are due for monthly {law}-FZ lookup"
|
||||
)
|
||||
|
||||
client = EisRegistryProcurementClient(
|
||||
source=source,
|
||||
law=law,
|
||||
proxies=proxies,
|
||||
)
|
||||
records: list[GenericParserItem] = []
|
||||
failed_lookups = 0
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=claim_source,
|
||||
)
|
||||
if attempt is None:
|
||||
continue
|
||||
records_before = len(records)
|
||||
try:
|
||||
records.extend(
|
||||
client.fetch_for_customer(
|
||||
inn=target.inn,
|
||||
organization_name=target.name,
|
||||
organization_id=target.organization_id,
|
||||
)
|
||||
)
|
||||
except HTTPClientError as exc:
|
||||
failed_lookups += 1
|
||||
finish_collection(attempt, records_count=0, error=exc)
|
||||
logger.info(
|
||||
"EIS %s-FZ lookup failed for customer_inn=%s: %s",
|
||||
law,
|
||||
target.inn,
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
finish_collection(
|
||||
attempt,
|
||||
records_count=len(records) - records_before,
|
||||
)
|
||||
|
||||
if failed_lookups == len(targets) and not records:
|
||||
raise ParserSourceSkipped(f"EIS {law}-FZ lookups failed for all targets")
|
||||
return records
|
||||
|
||||
|
||||
def _checko_unfair_supplier_items(
|
||||
*,
|
||||
company,
|
||||
target: RegistryLookupTarget,
|
||||
) -> list[GenericParserItem]:
|
||||
"""Convert Checko НедобПостЗап values into supplier-bound source records."""
|
||||
company_inn = _normalize_identifier(getattr(company, "inn", ""))
|
||||
company_ogrn = _normalize_identifier(getattr(company, "ogrn", ""))
|
||||
if target.inn and company_inn != target.inn:
|
||||
return []
|
||||
if not target.inn and target.ogrn and company_ogrn != target.ogrn:
|
||||
return []
|
||||
|
||||
records = []
|
||||
for item in getattr(company, "unfair_supplier", ()):
|
||||
registry_number = str(getattr(item, "registry_number", "") or "").strip()
|
||||
purchase_number = str(getattr(item, "purchase_number", "") or "").strip()
|
||||
stable_id = registry_number or purchase_number
|
||||
if not stable_id:
|
||||
stable_id = hashlib.sha256(
|
||||
repr(item).encode("utf-8", errors="replace")
|
||||
).hexdigest()
|
||||
customer = {
|
||||
"short_name": getattr(item, "customer_short_name", None),
|
||||
"full_name": getattr(item, "customer_full_name", None),
|
||||
"inn": getattr(item, "customer_inn", None),
|
||||
"kpp": getattr(item, "customer_kpp", None),
|
||||
}
|
||||
contract_price = getattr(item, "contract_price", None)
|
||||
records.append(
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
external_id=f"checko-unfair-supplier:{stable_id}",
|
||||
inn=target.inn,
|
||||
ogrn=target.ogrn,
|
||||
organisation_name=target.name,
|
||||
title=str(
|
||||
getattr(item, "purchase_description", None)
|
||||
or "Запись реестра недобросовестных поставщиков"
|
||||
),
|
||||
record_date=str(
|
||||
getattr(item, "publish_date", None)
|
||||
or getattr(item, "approval_date", None)
|
||||
or ""
|
||||
),
|
||||
amount=(
|
||||
Decimal(str(contract_price)) if contract_price is not None else None
|
||||
),
|
||||
status="included",
|
||||
payload={
|
||||
"provider": "checko",
|
||||
"organization_id": target.organization_id,
|
||||
"registry_number": registry_number,
|
||||
"publish_date": getattr(item, "publish_date", None),
|
||||
"approval_date": getattr(item, "approval_date", None),
|
||||
"purchase_number": purchase_number,
|
||||
"purchase_description": getattr(
|
||||
item,
|
||||
"purchase_description",
|
||||
None,
|
||||
),
|
||||
"contract_price": contract_price,
|
||||
"supplier": {
|
||||
"inn": target.inn,
|
||||
"ogrn": target.ogrn,
|
||||
"name": target.name,
|
||||
},
|
||||
"customer": customer,
|
||||
},
|
||||
)
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
def _fetch_checko_unfair_supplier_records( # noqa: C901
|
||||
*,
|
||||
limit: int | None,
|
||||
proxies: list[str] | None,
|
||||
organization_ids: list[str] | None = None,
|
||||
) -> list[GenericParserItem]:
|
||||
"""Fetch RNP entries through one Checko /company request per OПК organization."""
|
||||
api_key = getattr(settings, "CHECKO_API_KEY", "")
|
||||
if not api_key:
|
||||
raise ParserSourceSkipped("CHECKO_API_KEY is empty; RNP parser skipped")
|
||||
|
||||
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
||||
if resolved_limit <= 0:
|
||||
return []
|
||||
targets = _active_registry_lookup_targets(
|
||||
limit=resolved_limit,
|
||||
checko_source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
|
||||
organization_ids=organization_ids,
|
||||
)
|
||||
if not targets:
|
||||
raise ParserSourceSkipped(
|
||||
"no OПК registry organizations are due for monthly RNP lookup"
|
||||
)
|
||||
|
||||
checko_proxies = (
|
||||
proxies if getattr(settings, "CHECKO_USE_RUNTIME_PROXIES", False) else None
|
||||
)
|
||||
client = CheckoClient(api_key=api_key, proxies=checko_proxies, timeout=30)
|
||||
records: list[GenericParserItem] = []
|
||||
failed_lookups = 0
|
||||
rate_limited = False
|
||||
for target in targets:
|
||||
attempt = claim_monthly_collection(
|
||||
organization_id=target.organization_id,
|
||||
source=CheckoCollectionAttempt.Source.UNFAIR_SUPPLIERS,
|
||||
)
|
||||
if attempt is None:
|
||||
continue
|
||||
records_before = len(records)
|
||||
try:
|
||||
response = client.get_company(
|
||||
CompanyRequest(inn=target.inn or None, ogrn=target.ogrn or None)
|
||||
)
|
||||
except CheckoRateLimitError as exc:
|
||||
finish_collection(attempt, records_count=0, error=exc)
|
||||
failed_lookups += 1
|
||||
rate_limited = True
|
||||
logger.warning("Checko RNP lookup stopped: quota/rate limit reached")
|
||||
break
|
||||
except CheckoError as exc:
|
||||
finish_collection(attempt, records_count=0, error=exc)
|
||||
failed_lookups += 1
|
||||
logger.info(
|
||||
"Checko RNP lookup failed for target=%s: %s",
|
||||
target.inn or target.ogrn,
|
||||
exc,
|
||||
)
|
||||
else:
|
||||
if response.data is not None:
|
||||
records.extend(
|
||||
_checko_unfair_supplier_items(
|
||||
company=response.data,
|
||||
target=target,
|
||||
)
|
||||
)
|
||||
finish_collection(
|
||||
attempt,
|
||||
records_count=len(records) - records_before,
|
||||
)
|
||||
|
||||
if not records and (rate_limited or failed_lookups == len(targets)):
|
||||
raise ParserSourceSkipped("Checko RNP lookups failed for all targets")
|
||||
return records
|
||||
|
||||
|
||||
def _contract_party_payload(party) -> dict:
|
||||
if party is None:
|
||||
return {}
|
||||
@@ -2320,26 +2562,13 @@ def parse_all_sources(
|
||||
generic_results = {}
|
||||
if not getattr(settings, "CELERY_TASK_ALWAYS_EAGER", False):
|
||||
generic_results = {
|
||||
"procurements_44fz": parse_procurements_44fz.delay(proxies=proxies).id,
|
||||
"procurements_223fz": parse_procurements_223fz.delay(
|
||||
proxies=proxies
|
||||
"registry_enrichment": parse_registry_enrichment_sources.delay(
|
||||
limit=_resolve_registry_enrichment_limit(None),
|
||||
proxies=proxies,
|
||||
).id,
|
||||
"contracts": parse_contracts.delay(proxies=proxies).id,
|
||||
"registry_contracts": parse_registry_contracts.delay(
|
||||
proxies=proxies
|
||||
).id,
|
||||
"unfair_suppliers": parse_unfair_suppliers.delay(proxies=proxies).id,
|
||||
"fas_goz": parse_fas_goz_evasion.delay(proxies=proxies).id,
|
||||
"fns_financial": sync_fns_financial_reports.delay(proxies=proxies).id,
|
||||
"arbitration": parse_arbitration_cases.delay(proxies=proxies).id,
|
||||
"fedresurs_bankruptcy": parse_fedresurs_bankruptcy.delay(
|
||||
proxies=proxies
|
||||
).id,
|
||||
"registry_inspections": parse_registry_inspections.delay(
|
||||
proxies=proxies
|
||||
).id,
|
||||
"fstec": parse_fstec_registers.delay(proxies=proxies).id,
|
||||
"trudvsem": parse_trudvsem_vacancies.delay(proxies=proxies).id,
|
||||
}
|
||||
|
||||
results = {
|
||||
@@ -2992,23 +3221,40 @@ def parse_procurements_44fz(
|
||||
*,
|
||||
file_url: str | None = None,
|
||||
file_path: str | None = None,
|
||||
limit: int | None = None,
|
||||
proxies: list[str] | None = None,
|
||||
organization_ids: list[str] | None = None,
|
||||
requested_by_id: int | None = None,
|
||||
) -> dict:
|
||||
"""Парсинг официальной выдачи ЕИС 44-ФЗ в GenericParserRecord."""
|
||||
"""Addressed OПК lookup by default; explicit files remain a manual tool."""
|
||||
proxies = _resolve_proxies(proxies)
|
||||
if file_url or file_path:
|
||||
|
||||
def fetch_records():
|
||||
return _fetch_structured_records(
|
||||
source_key="procurements_44fz",
|
||||
file_url=file_url,
|
||||
file_path=file_path,
|
||||
proxies=proxies,
|
||||
)
|
||||
else:
|
||||
|
||||
def fetch_records():
|
||||
return _fetch_eis_registry_procurement_records(
|
||||
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
law="44",
|
||||
limit=limit,
|
||||
proxies=proxies,
|
||||
organization_ids=organization_ids,
|
||||
)
|
||||
|
||||
return _run_generic_parser(
|
||||
self,
|
||||
source_key="procurements_44fz",
|
||||
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
task_name="apps.parsers.tasks.parse_procurements_44fz",
|
||||
requested_by_id=requested_by_id,
|
||||
fetch_records=lambda: _fetch_structured_records(
|
||||
source_key="procurements_44fz",
|
||||
file_url=file_url,
|
||||
file_path=file_path,
|
||||
proxies=proxies,
|
||||
),
|
||||
fetch_records=fetch_records,
|
||||
)
|
||||
|
||||
|
||||
@@ -3018,23 +3264,40 @@ def parse_procurements_223fz(
|
||||
*,
|
||||
file_url: str | None = None,
|
||||
file_path: str | None = None,
|
||||
limit: int | None = None,
|
||||
proxies: list[str] | None = None,
|
||||
organization_ids: list[str] | None = None,
|
||||
requested_by_id: int | None = None,
|
||||
) -> dict:
|
||||
"""Парсинг официальной выдачи ЕИС 223-ФЗ в GenericParserRecord."""
|
||||
"""Addressed OПК lookup by default; explicit files remain a manual tool."""
|
||||
proxies = _resolve_proxies(proxies)
|
||||
if file_url or file_path:
|
||||
|
||||
def fetch_records():
|
||||
return _fetch_structured_records(
|
||||
source_key="procurements_223fz",
|
||||
file_url=file_url,
|
||||
file_path=file_path,
|
||||
proxies=proxies,
|
||||
)
|
||||
else:
|
||||
|
||||
def fetch_records():
|
||||
return _fetch_eis_registry_procurement_records(
|
||||
source=ParserLoadLog.Source.PROCUREMENTS_223FZ,
|
||||
law="223",
|
||||
limit=limit,
|
||||
proxies=proxies,
|
||||
organization_ids=organization_ids,
|
||||
)
|
||||
|
||||
return _run_generic_parser(
|
||||
self,
|
||||
source_key="procurements_223fz",
|
||||
source=ParserLoadLog.Source.PROCUREMENTS_223FZ,
|
||||
task_name="apps.parsers.tasks.parse_procurements_223fz",
|
||||
requested_by_id=requested_by_id,
|
||||
fetch_records=lambda: _fetch_structured_records(
|
||||
source_key="procurements_223fz",
|
||||
file_url=file_url,
|
||||
file_path=file_path,
|
||||
proxies=proxies,
|
||||
),
|
||||
fetch_records=fetch_records,
|
||||
)
|
||||
|
||||
|
||||
@@ -3093,23 +3356,38 @@ def parse_unfair_suppliers(
|
||||
*,
|
||||
file_url: str | None = None,
|
||||
file_path: str | None = None,
|
||||
limit: int | None = None,
|
||||
proxies: list[str] | None = None,
|
||||
organization_ids: list[str] | None = None,
|
||||
requested_by_id: int | None = None,
|
||||
) -> dict:
|
||||
"""Парсинг реестра недобросовестных поставщиков."""
|
||||
"""Checko RNP lookup by default; explicit files remain a manual tool."""
|
||||
proxies = _resolve_proxies(proxies)
|
||||
if file_url or file_path:
|
||||
|
||||
def fetch_records():
|
||||
return _fetch_structured_records(
|
||||
source_key="unfair_suppliers",
|
||||
file_url=file_url,
|
||||
file_path=file_path,
|
||||
proxies=proxies,
|
||||
)
|
||||
else:
|
||||
|
||||
def fetch_records():
|
||||
return _fetch_checko_unfair_supplier_records(
|
||||
limit=limit,
|
||||
proxies=proxies,
|
||||
organization_ids=organization_ids,
|
||||
)
|
||||
|
||||
return _run_generic_parser(
|
||||
self,
|
||||
source_key="unfair_suppliers",
|
||||
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
task_name="apps.parsers.tasks.parse_unfair_suppliers",
|
||||
requested_by_id=requested_by_id,
|
||||
fetch_records=lambda: _fetch_structured_records(
|
||||
source_key="unfair_suppliers",
|
||||
file_url=file_url,
|
||||
file_path=file_path,
|
||||
proxies=proxies,
|
||||
),
|
||||
fetch_records=fetch_records,
|
||||
)
|
||||
|
||||
|
||||
@@ -3241,11 +3519,13 @@ def parse_registry_enrichment_sources(
|
||||
"""
|
||||
Запустить daily-контур обогащения активных организаций из реестров.
|
||||
|
||||
Внутри остаются независимые задачи: одни забирают полный официальный реестр,
|
||||
другие делают lookup по ИНН/ОГРН активных организаций.
|
||||
Each addressed source advances through the next monthly registry batch.
|
||||
"""
|
||||
proxies = _resolve_proxies(proxies)
|
||||
resolved_limit = _resolve_registry_enrichment_limit(limit)
|
||||
tasks_to_run = {
|
||||
"procurements_44fz": parse_procurements_44fz,
|
||||
"procurements_223fz": parse_procurements_223fz,
|
||||
"contracts": parse_registry_contracts,
|
||||
"unfair_suppliers": parse_unfair_suppliers,
|
||||
"arbitration": parse_arbitration_cases,
|
||||
@@ -3259,8 +3539,15 @@ def parse_registry_enrichment_sources(
|
||||
"proxies": proxies,
|
||||
"requested_by_id": requested_by_id,
|
||||
}
|
||||
if key in {"contracts", "arbitration", "inspections"}:
|
||||
kwargs["limit"] = limit
|
||||
if key in {
|
||||
"procurements_44fz",
|
||||
"procurements_223fz",
|
||||
"contracts",
|
||||
"unfair_suppliers",
|
||||
"arbitration",
|
||||
"inspections",
|
||||
}:
|
||||
kwargs["limit"] = resolved_limit
|
||||
result = task.delay(**kwargs)
|
||||
results[key] = result.id
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user