fix(parsers): make source jobs resumable
All checks were successful
All checks were successful
This commit is contained in:
@@ -25,6 +25,8 @@ TRUDVSEM_SOURCE = "trudvsem"
|
||||
HH_SOURCE = "hh"
|
||||
SUPERJOB_SOURCE = "superjob"
|
||||
SUPPORTED_VACANCY_SOURCES = (TRUDVSEM_SOURCE, HH_SOURCE, SUPERJOB_SOURCE)
|
||||
ACTIVE_VACANCY_SOURCES = (TRUDVSEM_SOURCE,)
|
||||
DISABLED_VACANCY_SOURCES = (HH_SOURCE, SUPERJOB_SOURCE)
|
||||
|
||||
|
||||
class VacanciesClientError(HTTPClientError):
|
||||
@@ -297,21 +299,17 @@ class VacanciesClient:
|
||||
|
||||
clients: dict[str, VacancyProvider] = {
|
||||
TRUDVSEM_SOURCE: TrudvsemClient(proxies=self.proxies),
|
||||
HH_SOURCE: HHVacanciesClient(
|
||||
user_agent=self.hh_user_agent,
|
||||
proxies=self.proxies,
|
||||
),
|
||||
}
|
||||
if self.superjob_app_id:
|
||||
clients[SUPERJOB_SOURCE] = SuperJobVacanciesClient(
|
||||
app_id=self.superjob_app_id,
|
||||
proxies=self.proxies,
|
||||
)
|
||||
self._source_clients_cache = clients
|
||||
return clients
|
||||
|
||||
def _selected_sources(self) -> list[str]:
|
||||
selected = self.sources or list(SUPPORTED_VACANCY_SOURCES)
|
||||
selected = self.sources or list(ACTIVE_VACANCY_SOURCES)
|
||||
disabled = sorted(set(selected) & set(DISABLED_VACANCY_SOURCES))
|
||||
if disabled:
|
||||
raise VacanciesClientError(
|
||||
f"Disabled vacancy sources: {', '.join(disabled)}"
|
||||
)
|
||||
unknown = sorted(set(selected) - set(SUPPORTED_VACANCY_SOURCES))
|
||||
if unknown:
|
||||
raise VacanciesClientError(
|
||||
|
||||
@@ -540,7 +540,7 @@ class ParserRunRequestSerializer(serializers.Serializer):
|
||||
allow_empty=True,
|
||||
)
|
||||
vacancy_sources = serializers.ListField(
|
||||
child=serializers.ChoiceField(choices=["trudvsem", "hh", "superjob"]),
|
||||
child=serializers.ChoiceField(choices=["trudvsem"]),
|
||||
required=False,
|
||||
allow_empty=False,
|
||||
)
|
||||
|
||||
@@ -335,7 +335,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
SourceItemDefinition(
|
||||
code="trudvsem",
|
||||
title="Вакансии Работа России",
|
||||
description="Вакансии работодателей из Работа России, HH и SuperJob.",
|
||||
description="Вакансии работодателей из Работа России.",
|
||||
parser_source=ParserLoadLog.Source.TRUDVSEM,
|
||||
),
|
||||
),
|
||||
@@ -365,7 +365,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
|
||||
title="Новости СМИ",
|
||||
description="Загруженные упоминания организаций в СМИ с оценкой тональности.",
|
||||
order=110,
|
||||
task_names=(),
|
||||
task_names=("apps.parsers.tasks.parse_media_news",),
|
||||
source_items=(
|
||||
SourceItemDefinition(
|
||||
code="media_mentions",
|
||||
@@ -1160,6 +1160,20 @@ class SourceCardService:
|
||||
meta: dict[str, Any],
|
||||
kwargs: dict[str, Any],
|
||||
) -> dict[str, str]:
|
||||
if task_name == "apps.parsers.tasks.parse_trudvsem_vacancies":
|
||||
existing_job = (
|
||||
BackgroundJobService.get_queryset()
|
||||
.filter(task_name=task_name)
|
||||
.filter(cls._fresh_active_job_filter())
|
||||
.order_by("-created_at")
|
||||
.first()
|
||||
)
|
||||
if existing_job is not None:
|
||||
return {
|
||||
"task_id": existing_job.task_id,
|
||||
"task_name": existing_job.task_name,
|
||||
}
|
||||
|
||||
task_id = str(uuid.uuid4())
|
||||
BackgroundJobService.create_job(
|
||||
task_id=task_id,
|
||||
|
||||
@@ -285,15 +285,15 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
|
||||
key="trudvsem",
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
title="Вакансии",
|
||||
agency="Работа России / HH / SuperJob",
|
||||
data_scope="Вакансии работодателей из нескольких job-board источников",
|
||||
agency="Работа России",
|
||||
data_scope="Вакансии работодателей из официального источника Работа России",
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
|
||||
upstream_url="https://opendata.trudvsem.ru/api/v1/vacancies",
|
||||
access_method="public_api",
|
||||
parser_strategy="multi_source_vacancies_api",
|
||||
parser_strategy="incremental_trudvsem_api",
|
||||
source_notes=(
|
||||
"Internal source key remains trudvsem for backward compatibility; "
|
||||
"payload.vacancy_source distinguishes trudvsem, hh and superjob."
|
||||
"Пакетная обработка организаций с промежуточным сохранением, "
|
||||
"прогрессом и возобновлением после остановки."
|
||||
),
|
||||
api_route="trudvsem/vacancies",
|
||||
),
|
||||
|
||||
@@ -16,6 +16,7 @@ from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.core.services import BackgroundJobService
|
||||
from apps.core.tasks import PeriodicTask as CorePeriodicTask
|
||||
from apps.parsers.checko_collection import (
|
||||
@@ -73,7 +74,12 @@ from apps.parsers.source_registry import PARSER_SOURCES
|
||||
from celery import shared_task
|
||||
from django.conf import settings
|
||||
from django.db.models import Q
|
||||
from organizations.models import Organization as SourceOrganization
|
||||
from organizations.models import (
|
||||
Organization as SourceOrganization,
|
||||
)
|
||||
from organizations.models import (
|
||||
OrganizationSourceRecord,
|
||||
)
|
||||
from organizations.services import (
|
||||
normalize_organization_name as normalize_identity_name,
|
||||
)
|
||||
@@ -141,30 +147,9 @@ class FNSApiFetchResult:
|
||||
|
||||
|
||||
VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION = 100
|
||||
VACANCY_REGISTRY_TEXT_SEARCH_MAX_PAGES_PER_ORGANIZATION = 1
|
||||
VACANCY_EMPLOYER_WORD_RE = re.compile(r"[0-9A-Za-zА-Яа-яЁё]+")
|
||||
VACANCY_EMPLOYER_IGNORED_WORDS = {
|
||||
"ао",
|
||||
"акционерное",
|
||||
"государственное",
|
||||
"зао",
|
||||
"индивидуальный",
|
||||
"ип",
|
||||
"муниципальное",
|
||||
"нао",
|
||||
"некоммерческая",
|
||||
"оао",
|
||||
"общество",
|
||||
"ограниченной",
|
||||
"ооо",
|
||||
"ответственностью",
|
||||
"пао",
|
||||
"предприниматель",
|
||||
"публичное",
|
||||
"с",
|
||||
"унитарное",
|
||||
"фгуп",
|
||||
}
|
||||
VACANCY_REGISTRY_ORGANIZATIONS_PER_TASK = 25
|
||||
VACANCY_TASK_NAME = "apps.parsers.tasks.parse_trudvsem_vacancies"
|
||||
TRUDVSEM_VACANCY_SOURCE = "trudvsem"
|
||||
|
||||
|
||||
def _resolve_lookup_limit(
|
||||
@@ -3787,7 +3772,427 @@ def cleanup_stale_parser_loads(
|
||||
}
|
||||
|
||||
|
||||
@shared_task(bind=True)
|
||||
def _normalize_trudvsem_vacancy_sources(
|
||||
vacancy_sources: list[str] | None,
|
||||
) -> list[str]:
|
||||
"""Оставить единственный включённый источник вакансий."""
|
||||
selected = vacancy_sources or [TRUDVSEM_VACANCY_SOURCE]
|
||||
disabled = sorted(set(selected) - {TRUDVSEM_VACANCY_SOURCE})
|
||||
if disabled:
|
||||
raise ValueError("Поддерживается только источник trudvsem")
|
||||
return [TRUDVSEM_VACANCY_SOURCE]
|
||||
|
||||
|
||||
def _vacancy_targets_signature(targets: list[RegistryLookupTarget]) -> str:
|
||||
"""Вернуть безопасный отпечаток порядка организаций для возобновления."""
|
||||
digest = hashlib.sha256()
|
||||
for target in targets:
|
||||
digest.update(target.organization_id.encode("utf-8"))
|
||||
digest.update(b"\n")
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _vacancy_batch_records_count(batch_id: int) -> int:
|
||||
return OrganizationSourceRecord.objects.filter(
|
||||
source=TRUDVSEM_VACANCY_SOURCE,
|
||||
load_batch=batch_id,
|
||||
).count()
|
||||
|
||||
|
||||
def _find_resumable_vacancy_job(
|
||||
*,
|
||||
exclude_task_id: str,
|
||||
targets_signature: str,
|
||||
targets_count: int,
|
||||
):
|
||||
"""Найти последний совместимый остановленный проход вакансий."""
|
||||
candidates = (
|
||||
BackgroundJobService.get_queryset()
|
||||
.filter(
|
||||
task_name=VACANCY_TASK_NAME,
|
||||
status__in=[JobStatus.REVOKED, JobStatus.FAILURE],
|
||||
)
|
||||
.exclude(task_id=exclude_task_id)
|
||||
.order_by("-updated_at")[:20]
|
||||
)
|
||||
for candidate in candidates:
|
||||
meta = candidate.meta or {}
|
||||
next_offset = int(meta.get("next_offset") or 0)
|
||||
batch_id = meta.get("batch_id")
|
||||
if (
|
||||
meta.get("targets_signature") != targets_signature
|
||||
or int(meta.get("total_organizations") or 0) != targets_count
|
||||
or next_offset <= 0
|
||||
or next_offset >= targets_count
|
||||
or batch_id is None
|
||||
):
|
||||
continue
|
||||
load = ParserLoadLog.objects.filter(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=batch_id,
|
||||
).first()
|
||||
if load is not None:
|
||||
return candidate, load
|
||||
return None, None
|
||||
|
||||
|
||||
def _update_vacancy_registry_checkpoint(
|
||||
*,
|
||||
job,
|
||||
load_log: ParserLoadLog,
|
||||
batch_id: int,
|
||||
processed: int,
|
||||
total: int,
|
||||
failed: int,
|
||||
targets_signature: str,
|
||||
) -> int:
|
||||
"""Зафиксировать прогресс после полностью обработанной организации."""
|
||||
saved_count = _vacancy_batch_records_count(batch_id)
|
||||
progress = 100 if total == 0 else min(99, round(processed * 100 / total))
|
||||
job.meta = {
|
||||
**(job.meta or {}),
|
||||
"batch_id": batch_id,
|
||||
"next_offset": processed,
|
||||
"total_organizations": total,
|
||||
"failed_organizations": failed,
|
||||
"saved_records": saved_count,
|
||||
"targets_signature": targets_signature,
|
||||
}
|
||||
job.progress = progress
|
||||
job.progress_message = (
|
||||
f"Обработано организаций: {processed} из {total}; "
|
||||
f"сохранено вакансий: {saved_count}; ошибок: {failed}"
|
||||
)
|
||||
job.save(update_fields=["meta", "progress", "progress_message", "updated_at"])
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.IN_PROGRESS,
|
||||
records_count=saved_count,
|
||||
error_message="",
|
||||
)
|
||||
return saved_count
|
||||
|
||||
|
||||
def _finish_revoked_vacancy_registry_run(
|
||||
*,
|
||||
job,
|
||||
load_log: ParserLoadLog,
|
||||
batch_id: int,
|
||||
processed: int,
|
||||
total: int,
|
||||
failed: int,
|
||||
) -> dict:
|
||||
"""Закрыть отменённый проход, сохранив уже записанный результат."""
|
||||
saved_count = _vacancy_batch_records_count(batch_id)
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.SKIPPED,
|
||||
records_count=saved_count,
|
||||
error_message=(
|
||||
f"Остановлено после {processed} из {total} организаций; "
|
||||
"сохранённые записи не удалены"
|
||||
),
|
||||
)
|
||||
return {
|
||||
"batch_id": batch_id,
|
||||
"saved": saved_count,
|
||||
"processed_organizations": processed,
|
||||
"failed_organizations": failed,
|
||||
"status": "revoked",
|
||||
"resumed": bool((job.meta or {}).get("resumed")),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class VacancyRegistryRunState:
|
||||
"""Состояние одного возобновляемого прохода вакансий."""
|
||||
|
||||
task_id: str
|
||||
targets: list[RegistryLookupTarget]
|
||||
targets_signature: str
|
||||
job: BackgroundJob
|
||||
load_log: ParserLoadLog
|
||||
batch_id: int
|
||||
next_offset: int
|
||||
failed: int
|
||||
resumed: bool
|
||||
|
||||
@property
|
||||
def total(self) -> int:
|
||||
return len(self.targets)
|
||||
|
||||
|
||||
def _prepare_vacancy_registry_run(
|
||||
self,
|
||||
*,
|
||||
registry_organization_limit: int | None,
|
||||
requested_by_id: int | None,
|
||||
run_task_id: str | None,
|
||||
) -> VacancyRegistryRunState:
|
||||
"""Создать новый проход или восстановить совместимый checkpoint."""
|
||||
task_id = run_task_id or self.request.id or str(uuid.uuid4())
|
||||
targets = _active_registry_vacancy_targets(limit=registry_organization_limit)
|
||||
total = len(targets)
|
||||
targets_signature = _vacancy_targets_signature(targets)
|
||||
resumed = False
|
||||
|
||||
job = BackgroundJobService.get_by_task_id_or_none(task_id)
|
||||
if run_task_id is not None:
|
||||
if job is None:
|
||||
raise RuntimeError("Не найдена родительская задача вакансий")
|
||||
meta = job.meta or {}
|
||||
batch_id = int(meta["batch_id"])
|
||||
load_log = ParserLoadLog.objects.get(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=batch_id,
|
||||
)
|
||||
if meta.get("targets_signature") != targets_signature:
|
||||
meta = {
|
||||
**meta,
|
||||
"next_offset": 0,
|
||||
"failed_organizations": 0,
|
||||
"total_organizations": total,
|
||||
"targets_signature": targets_signature,
|
||||
}
|
||||
job.meta = meta
|
||||
job.save(update_fields=["meta", "updated_at"])
|
||||
resumed = bool(meta.get("resumed"))
|
||||
else:
|
||||
resumable_job, resumable_load = _find_resumable_vacancy_job(
|
||||
exclude_task_id=task_id,
|
||||
targets_signature=targets_signature,
|
||||
targets_count=total,
|
||||
)
|
||||
if resumable_job is not None and resumable_load is not None:
|
||||
resumable_meta = resumable_job.meta or {}
|
||||
batch_id = int(resumable_meta["batch_id"])
|
||||
load_log = resumable_load
|
||||
next_offset = int(resumable_meta["next_offset"])
|
||||
failed = int(resumable_meta.get("failed_organizations") or 0)
|
||||
resumed = True
|
||||
ParserLoadLogService.update(
|
||||
load_log,
|
||||
status=ParserLoadLog.Status.IN_PROGRESS,
|
||||
records_count=_vacancy_batch_records_count(batch_id),
|
||||
error_message="",
|
||||
)
|
||||
else:
|
||||
(
|
||||
load_log,
|
||||
batch_id,
|
||||
) = ParserLoadLogService.create_load_log_with_next_batch_id(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
status=ParserLoadLog.Status.IN_PROGRESS,
|
||||
)
|
||||
next_offset = 0
|
||||
failed = 0
|
||||
|
||||
job = _get_or_create_background_job(
|
||||
task_id=task_id,
|
||||
task_name=VACANCY_TASK_NAME,
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=batch_id,
|
||||
requested_by_id=requested_by_id,
|
||||
meta={
|
||||
"source_key": TRUDVSEM_VACANCY_SOURCE,
|
||||
"next_offset": next_offset,
|
||||
"total_organizations": total,
|
||||
"failed_organizations": failed,
|
||||
"saved_records": _vacancy_batch_records_count(batch_id),
|
||||
"targets_signature": targets_signature,
|
||||
"resumed": resumed,
|
||||
"resumed_from_task_id": (
|
||||
resumable_job.task_id if resumable_job is not None else None
|
||||
),
|
||||
},
|
||||
)
|
||||
job.mark_started()
|
||||
_update_vacancy_registry_checkpoint(
|
||||
job=job,
|
||||
load_log=load_log,
|
||||
batch_id=batch_id,
|
||||
processed=next_offset,
|
||||
total=total,
|
||||
failed=failed,
|
||||
targets_signature=targets_signature,
|
||||
)
|
||||
|
||||
meta = job.meta or {}
|
||||
return VacancyRegistryRunState(
|
||||
task_id=task_id,
|
||||
targets=targets,
|
||||
targets_signature=targets_signature,
|
||||
job=job,
|
||||
load_log=load_log,
|
||||
batch_id=batch_id,
|
||||
next_offset=int(meta.get("next_offset") or 0),
|
||||
failed=int(meta.get("failed_organizations") or 0),
|
||||
resumed=resumed,
|
||||
)
|
||||
|
||||
|
||||
def _vacancy_registry_run_is_revoked(state: VacancyRegistryRunState) -> bool:
|
||||
state.job.refresh_from_db(fields=["status", "meta"])
|
||||
return state.job.status == JobStatus.REVOKED
|
||||
|
||||
|
||||
def _revoked_vacancy_registry_result(state: VacancyRegistryRunState) -> dict:
|
||||
return _finish_revoked_vacancy_registry_run(
|
||||
job=state.job,
|
||||
load_log=state.load_log,
|
||||
batch_id=state.batch_id,
|
||||
processed=state.next_offset,
|
||||
total=state.total,
|
||||
failed=state.failed,
|
||||
)
|
||||
|
||||
|
||||
def _process_vacancy_registry_chunk(
|
||||
state: VacancyRegistryRunState,
|
||||
*,
|
||||
limit: int,
|
||||
proxies: list[str] | None,
|
||||
) -> bool:
|
||||
"""Обработать ограниченный блок; вернуть True при отмене."""
|
||||
chunk_end = min(
|
||||
state.total,
|
||||
state.next_offset + max(1, VACANCY_REGISTRY_ORGANIZATIONS_PER_TASK),
|
||||
)
|
||||
|
||||
with VacanciesClient(
|
||||
proxies=proxies,
|
||||
sources=[TRUDVSEM_VACANCY_SOURCE],
|
||||
) as client:
|
||||
for index in range(state.next_offset, chunk_end):
|
||||
if _vacancy_registry_run_is_revoked(state):
|
||||
return True
|
||||
|
||||
target = state.targets[index]
|
||||
try:
|
||||
records = _fetch_registry_target_vacancy_records(
|
||||
client,
|
||||
target,
|
||||
page_size=max(1, limit),
|
||||
)
|
||||
if records:
|
||||
GenericParserRecordService.save_records(
|
||||
records,
|
||||
batch_id=state.batch_id,
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
)
|
||||
except Exception as exc:
|
||||
state.failed += 1
|
||||
logger.warning(
|
||||
"Vacancy fetch failed for registry target %d of %d: %s",
|
||||
index + 1,
|
||||
state.total,
|
||||
exc,
|
||||
)
|
||||
|
||||
state.next_offset = index + 1
|
||||
_update_vacancy_registry_checkpoint(
|
||||
job=state.job,
|
||||
load_log=state.load_log,
|
||||
batch_id=state.batch_id,
|
||||
processed=state.next_offset,
|
||||
total=state.total,
|
||||
failed=state.failed,
|
||||
targets_signature=state.targets_signature,
|
||||
)
|
||||
return _vacancy_registry_run_is_revoked(state)
|
||||
|
||||
|
||||
def _queue_or_finish_vacancy_registry_run(
|
||||
state: VacancyRegistryRunState,
|
||||
*,
|
||||
limit: int,
|
||||
registry_organization_limit: int | None,
|
||||
proxies: list[str] | None,
|
||||
requested_by_id: int | None,
|
||||
) -> dict:
|
||||
"""Передать следующий блок в очередь или закрыть проход."""
|
||||
saved_count = _vacancy_batch_records_count(state.batch_id)
|
||||
if state.next_offset < state.total:
|
||||
try:
|
||||
parse_trudvsem_vacancies.apply_async(
|
||||
kwargs={
|
||||
"limit": limit,
|
||||
"vacancy_sources": [TRUDVSEM_VACANCY_SOURCE],
|
||||
"registry_organizations_only": True,
|
||||
"registry_organization_limit": registry_organization_limit,
|
||||
"proxies": proxies,
|
||||
"requested_by_id": requested_by_id,
|
||||
"_vacancy_run_task_id": state.task_id,
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
ParserLoadLogService.mark_failed(state.load_log, str(exc))
|
||||
state.job.fail(error=str(exc))
|
||||
raise
|
||||
return {
|
||||
"batch_id": state.batch_id,
|
||||
"saved": saved_count,
|
||||
"processed_organizations": state.next_offset,
|
||||
"failed_organizations": state.failed,
|
||||
"status": "in_progress",
|
||||
"resumed": state.resumed,
|
||||
}
|
||||
|
||||
result = {
|
||||
"batch_id": state.batch_id,
|
||||
"saved": saved_count,
|
||||
"processed_organizations": state.next_offset,
|
||||
"failed_organizations": state.failed,
|
||||
"status": "success",
|
||||
"resumed": state.resumed,
|
||||
}
|
||||
if state.total > 0 and state.failed >= state.total:
|
||||
message = "Не удалось обработать ни одной организации в Работа России"
|
||||
result["status"] = "failure"
|
||||
ParserLoadLogService.mark_failed(state.load_log, message)
|
||||
state.job.fail(error=message)
|
||||
return result
|
||||
|
||||
ParserLoadLogService.update(
|
||||
state.load_log,
|
||||
status=ParserLoadLog.Status.SUCCESS,
|
||||
records_count=saved_count,
|
||||
error_message="",
|
||||
)
|
||||
state.job.complete(result=result)
|
||||
return result
|
||||
|
||||
|
||||
def _run_incremental_registry_vacancies(
|
||||
self,
|
||||
*,
|
||||
limit: int,
|
||||
registry_organization_limit: int | None,
|
||||
proxies: list[str] | None,
|
||||
requested_by_id: int | None,
|
||||
run_task_id: str | None,
|
||||
) -> dict:
|
||||
"""Обрабатывать реестр короткими возобновляемыми Celery-проходами."""
|
||||
state = _prepare_vacancy_registry_run(
|
||||
self,
|
||||
registry_organization_limit=registry_organization_limit,
|
||||
requested_by_id=requested_by_id,
|
||||
run_task_id=run_task_id,
|
||||
)
|
||||
if _vacancy_registry_run_is_revoked(state):
|
||||
return _revoked_vacancy_registry_result(state)
|
||||
if _process_vacancy_registry_chunk(state, limit=limit, proxies=proxies):
|
||||
return _revoked_vacancy_registry_result(state)
|
||||
return _queue_or_finish_vacancy_registry_run(
|
||||
state,
|
||||
limit=limit,
|
||||
registry_organization_limit=registry_organization_limit,
|
||||
proxies=proxies,
|
||||
requested_by_id=requested_by_id,
|
||||
)
|
||||
|
||||
|
||||
@shared_task(bind=True, acks_late=True, reject_on_worker_lost=True)
|
||||
def parse_trudvsem_vacancies(
|
||||
self,
|
||||
*,
|
||||
@@ -3801,14 +4206,50 @@ def parse_trudvsem_vacancies(
|
||||
registry_organization_limit: int | None = None,
|
||||
proxies: list[str] | None = None,
|
||||
requested_by_id: int | None = None,
|
||||
_vacancy_run_task_id: str | None = None,
|
||||
) -> dict:
|
||||
"""Парсинг вакансий по активным организациям реестров."""
|
||||
vacancy_sources = _normalize_trudvsem_vacancy_sources(vacancy_sources)
|
||||
proxies = _resolve_proxies(proxies)
|
||||
if _should_fetch_registry_organization_vacancies(
|
||||
registry_organizations_only=registry_organizations_only,
|
||||
region_code=region_code,
|
||||
company_inn=company_inn,
|
||||
text=text,
|
||||
):
|
||||
try:
|
||||
return _run_incremental_registry_vacancies(
|
||||
self,
|
||||
limit=limit,
|
||||
registry_organization_limit=registry_organization_limit,
|
||||
proxies=proxies,
|
||||
requested_by_id=requested_by_id,
|
||||
run_task_id=_vacancy_run_task_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
task_id = _vacancy_run_task_id or self.request.id
|
||||
job = (
|
||||
BackgroundJobService.get_by_task_id_or_none(task_id)
|
||||
if task_id
|
||||
else None
|
||||
)
|
||||
if job is not None and not job.is_finished:
|
||||
batch_id = (job.meta or {}).get("batch_id")
|
||||
if batch_id is not None:
|
||||
load_log = ParserLoadLog.objects.filter(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=batch_id,
|
||||
).first()
|
||||
if load_log is not None:
|
||||
ParserLoadLogService.mark_failed(load_log, str(exc))
|
||||
job.fail(error=str(exc))
|
||||
raise
|
||||
|
||||
return _run_generic_parser(
|
||||
self,
|
||||
source_key="trudvsem",
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
|
||||
task_name=VACANCY_TASK_NAME,
|
||||
requested_by_id=requested_by_id,
|
||||
fetch_records=lambda: _fetch_vacancy_records(
|
||||
proxies=proxies,
|
||||
@@ -3818,8 +4259,6 @@ def parse_trudvsem_vacancies(
|
||||
company_inn=company_inn,
|
||||
text=text,
|
||||
vacancy_sources=vacancy_sources,
|
||||
registry_organizations_only=registry_organizations_only,
|
||||
registry_organization_limit=registry_organization_limit,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -3833,26 +4272,9 @@ def _fetch_vacancy_records(
|
||||
company_inn: str | None,
|
||||
text: str | None,
|
||||
vacancy_sources: list[str] | None,
|
||||
registry_organizations_only: bool,
|
||||
registry_organization_limit: int | None,
|
||||
) -> list[GenericParserItem]:
|
||||
if _should_fetch_registry_organization_vacancies(
|
||||
registry_organizations_only=registry_organizations_only,
|
||||
region_code=region_code,
|
||||
company_inn=company_inn,
|
||||
text=text,
|
||||
):
|
||||
return _fetch_registry_organization_vacancy_records(
|
||||
proxies=proxies,
|
||||
limit=limit,
|
||||
vacancy_sources=vacancy_sources,
|
||||
registry_organization_limit=registry_organization_limit,
|
||||
)
|
||||
|
||||
with VacanciesClient(
|
||||
proxies=proxies,
|
||||
superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""),
|
||||
hh_user_agent=getattr(settings, "HH_USER_AGENT", ""),
|
||||
sources=vacancy_sources,
|
||||
) as client:
|
||||
return client.fetch_vacancies(
|
||||
@@ -3914,60 +4336,11 @@ def _fetch_registry_target_vacancy_records(
|
||||
*,
|
||||
page_size: int,
|
||||
) -> list[GenericParserItem]:
|
||||
iter_source_clients = getattr(client, "iter_source_clients", None)
|
||||
if iter_source_clients is None:
|
||||
return _fetch_registry_target_source_vacancy_records(
|
||||
client,
|
||||
target,
|
||||
page_size=page_size,
|
||||
company_inn=target.inn,
|
||||
)
|
||||
|
||||
records: list[GenericParserItem] = []
|
||||
errors: list[str] = []
|
||||
attempts = 0
|
||||
|
||||
for source, source_client in iter_source_clients():
|
||||
if getattr(source_client, "supports_company_inn", False):
|
||||
kwargs = {"company_inn": target.inn}
|
||||
else:
|
||||
if not target.name:
|
||||
logger.info(
|
||||
"Vacancy source %s is skipped for registry organization %s: "
|
||||
"empty organization name",
|
||||
source,
|
||||
target.organization_id,
|
||||
)
|
||||
continue
|
||||
kwargs = {"text": _vacancy_registry_text_query(target)}
|
||||
|
||||
attempts += 1
|
||||
try:
|
||||
source_records = _fetch_registry_target_source_vacancy_records(
|
||||
source_client,
|
||||
target,
|
||||
page_size=page_size,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Vacancy source %s failed for registry organization %s (%s): %s",
|
||||
source,
|
||||
target.organization_id,
|
||||
target.inn,
|
||||
exc,
|
||||
)
|
||||
errors.append(f"{source}: {exc}")
|
||||
continue
|
||||
|
||||
records.extend(source_records)
|
||||
|
||||
if errors and not records and attempts:
|
||||
raise RuntimeError(
|
||||
"All vacancy sources failed for registry organization "
|
||||
f"{target.organization_id} ({target.inn}); first error: {errors[0]}"
|
||||
)
|
||||
return records
|
||||
return _fetch_registry_target_source_vacancy_records(
|
||||
client,
|
||||
target,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
|
||||
def _fetch_registry_target_source_vacancy_records(
|
||||
@@ -3975,155 +4348,28 @@ def _fetch_registry_target_source_vacancy_records(
|
||||
target: RegistryLookupTarget,
|
||||
*,
|
||||
page_size: int,
|
||||
company_inn: str | None = None,
|
||||
text: str | None = None,
|
||||
) -> list[GenericParserItem]:
|
||||
records: list[GenericParserItem] = []
|
||||
offset = 0
|
||||
filter_by_employer_name = company_inn is None
|
||||
max_pages = (
|
||||
VACANCY_REGISTRY_TEXT_SEARCH_MAX_PAGES_PER_ORGANIZATION
|
||||
if filter_by_employer_name
|
||||
else VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION
|
||||
)
|
||||
for _ in range(max_pages):
|
||||
for _ in range(VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION):
|
||||
page_records = source_client.fetch_vacancies(
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
company_inn=company_inn,
|
||||
text=text,
|
||||
company_inn=target.inn,
|
||||
)
|
||||
records.extend(
|
||||
_attach_registry_vacancy_target(record, target) for record in page_records
|
||||
)
|
||||
if filter_by_employer_name:
|
||||
matched_records = [
|
||||
record
|
||||
for record in page_records
|
||||
if _vacancy_record_matches_registry_target(record, target)
|
||||
]
|
||||
else:
|
||||
matched_records = page_records
|
||||
records.extend(matched_records)
|
||||
if len(page_records) < page_size:
|
||||
return records
|
||||
offset += page_size
|
||||
|
||||
if filter_by_employer_name:
|
||||
return records
|
||||
|
||||
raise RuntimeError(
|
||||
"Vacancy registry organization page limit reached "
|
||||
f"for organization {target.organization_id} ({target.inn})"
|
||||
)
|
||||
|
||||
|
||||
def _vacancy_record_matches_registry_target(
|
||||
record: GenericParserItem,
|
||||
target: RegistryLookupTarget,
|
||||
) -> bool:
|
||||
target_key = _vacancy_employer_match_key(target.name)
|
||||
employer_key = _vacancy_employer_match_key(_vacancy_record_employer_name(record))
|
||||
if not target_key or not employer_key:
|
||||
return False
|
||||
if target_key == employer_key:
|
||||
return True
|
||||
if min(len(target_key), len(employer_key)) < 8:
|
||||
return False
|
||||
return target_key in employer_key or employer_key in target_key
|
||||
|
||||
|
||||
def _vacancy_registry_text_query(target: RegistryLookupTarget) -> str:
|
||||
return _vacancy_employer_match_key(target.name) or target.name
|
||||
|
||||
|
||||
def _vacancy_record_employer_name(record: GenericParserItem) -> str:
|
||||
if record.organisation_name:
|
||||
return record.organisation_name
|
||||
|
||||
payload = record.payload if isinstance(record.payload, dict) else {}
|
||||
for key in ("employer", "company"):
|
||||
nested = payload.get(key)
|
||||
if isinstance(nested, dict) and nested.get("name"):
|
||||
return str(nested["name"])
|
||||
for key in ("firm_name", "company_name", "organisation_name"):
|
||||
value = payload.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
return ""
|
||||
|
||||
|
||||
def _vacancy_employer_match_key(name: str) -> str:
|
||||
words = []
|
||||
for match in VACANCY_EMPLOYER_WORD_RE.finditer(name.casefold().replace("ё", "е")):
|
||||
word = match.group(0)
|
||||
if word not in VACANCY_EMPLOYER_IGNORED_WORDS:
|
||||
words.append(word)
|
||||
return " ".join(words)
|
||||
|
||||
|
||||
def _fetch_registry_organization_vacancy_records(
|
||||
*,
|
||||
proxies: list[str] | None,
|
||||
limit: int,
|
||||
vacancy_sources: list[str] | None,
|
||||
registry_organization_limit: int | None,
|
||||
) -> list[GenericParserItem]:
|
||||
targets = _active_registry_vacancy_targets(limit=registry_organization_limit)
|
||||
if not targets:
|
||||
logger.info("No active registry organizations for vacancies sync")
|
||||
return []
|
||||
|
||||
page_size = max(1, limit)
|
||||
records: list[GenericParserItem] = []
|
||||
errors: list[str] = []
|
||||
successful_fetches = 0
|
||||
with VacanciesClient(
|
||||
proxies=proxies,
|
||||
superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""),
|
||||
hh_user_agent=getattr(settings, "HH_USER_AGENT", ""),
|
||||
sources=vacancy_sources,
|
||||
) as client:
|
||||
for target in targets:
|
||||
try:
|
||||
organization_records = _fetch_registry_target_vacancy_records(
|
||||
client,
|
||||
target,
|
||||
page_size=page_size,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Vacancy fetch failed for registry organization %s (%s): %s",
|
||||
target.organization_id,
|
||||
target.inn,
|
||||
exc,
|
||||
)
|
||||
errors.append(f"{target.inn}: {exc}")
|
||||
continue
|
||||
|
||||
successful_fetches += 1
|
||||
records.extend(
|
||||
_attach_registry_vacancy_target(record, target)
|
||||
for record in organization_records
|
||||
)
|
||||
|
||||
if errors and successful_fetches == 0:
|
||||
raise RuntimeError(
|
||||
"All registry organization vacancy fetches failed; "
|
||||
f"first error: {errors[0]}"
|
||||
)
|
||||
if errors:
|
||||
logger.warning(
|
||||
"Vacancy registry organization sync completed with %d failed "
|
||||
"organizations",
|
||||
len(errors),
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Fetched %d vacancy records for %d active registry organizations",
|
||||
len(records),
|
||||
len(targets),
|
||||
)
|
||||
return records
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# FNS Tasks (File Watch & Processing)
|
||||
# =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user