fix(parsers): make source jobs resumable
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 1m11s
CI/CD Pipeline / Build and Push Images (push) Successful in 19s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 24s

This commit is contained in:
2026-08-14 17:02:56 +02:00
parent a0eb0a5303
commit 31eaa7e907
14 changed files with 884 additions and 476 deletions

View File

@@ -25,6 +25,8 @@ TRUDVSEM_SOURCE = "trudvsem"
HH_SOURCE = "hh"
SUPERJOB_SOURCE = "superjob"
SUPPORTED_VACANCY_SOURCES = (TRUDVSEM_SOURCE, HH_SOURCE, SUPERJOB_SOURCE)
ACTIVE_VACANCY_SOURCES = (TRUDVSEM_SOURCE,)
DISABLED_VACANCY_SOURCES = (HH_SOURCE, SUPERJOB_SOURCE)
class VacanciesClientError(HTTPClientError):
@@ -297,21 +299,17 @@ class VacanciesClient:
clients: dict[str, VacancyProvider] = {
TRUDVSEM_SOURCE: TrudvsemClient(proxies=self.proxies),
HH_SOURCE: HHVacanciesClient(
user_agent=self.hh_user_agent,
proxies=self.proxies,
),
}
if self.superjob_app_id:
clients[SUPERJOB_SOURCE] = SuperJobVacanciesClient(
app_id=self.superjob_app_id,
proxies=self.proxies,
)
self._source_clients_cache = clients
return clients
def _selected_sources(self) -> list[str]:
selected = self.sources or list(SUPPORTED_VACANCY_SOURCES)
selected = self.sources or list(ACTIVE_VACANCY_SOURCES)
disabled = sorted(set(selected) & set(DISABLED_VACANCY_SOURCES))
if disabled:
raise VacanciesClientError(
f"Disabled vacancy sources: {', '.join(disabled)}"
)
unknown = sorted(set(selected) - set(SUPPORTED_VACANCY_SOURCES))
if unknown:
raise VacanciesClientError(

View File

@@ -540,7 +540,7 @@ class ParserRunRequestSerializer(serializers.Serializer):
allow_empty=True,
)
vacancy_sources = serializers.ListField(
child=serializers.ChoiceField(choices=["trudvsem", "hh", "superjob"]),
child=serializers.ChoiceField(choices=["trudvsem"]),
required=False,
allow_empty=False,
)

View File

@@ -335,7 +335,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
SourceItemDefinition(
code="trudvsem",
title="Вакансии Работа России",
description="Вакансии работодателей из Работа России, HH и SuperJob.",
description="Вакансии работодателей из Работа России.",
parser_source=ParserLoadLog.Source.TRUDVSEM,
),
),
@@ -365,7 +365,7 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = (
title="Новости СМИ",
description="Загруженные упоминания организаций в СМИ с оценкой тональности.",
order=110,
task_names=(),
task_names=("apps.parsers.tasks.parse_media_news",),
source_items=(
SourceItemDefinition(
code="media_mentions",
@@ -1160,6 +1160,20 @@ class SourceCardService:
meta: dict[str, Any],
kwargs: dict[str, Any],
) -> dict[str, str]:
if task_name == "apps.parsers.tasks.parse_trudvsem_vacancies":
existing_job = (
BackgroundJobService.get_queryset()
.filter(task_name=task_name)
.filter(cls._fresh_active_job_filter())
.order_by("-created_at")
.first()
)
if existing_job is not None:
return {
"task_id": existing_job.task_id,
"task_name": existing_job.task_name,
}
task_id = str(uuid.uuid4())
BackgroundJobService.create_job(
task_id=task_id,

View File

@@ -285,15 +285,15 @@ PARSER_SOURCES: dict[str, ParserSourceDescriptor] = {
key="trudvsem",
source=ParserLoadLog.Source.TRUDVSEM,
title="Вакансии",
agency="Работа России / HH / SuperJob",
data_scope="Вакансии работодателей из нескольких job-board источников",
agency="Работа России",
data_scope="Вакансии работодателей из официального источника Работа России",
task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
upstream_url="https://opendata.trudvsem.ru/api/v1/vacancies",
access_method="public_api",
parser_strategy="multi_source_vacancies_api",
parser_strategy="incremental_trudvsem_api",
source_notes=(
"Internal source key remains trudvsem for backward compatibility; "
"payload.vacancy_source distinguishes trudvsem, hh and superjob."
"Пакетная обработка организаций с промежуточным сохранением, "
"прогрессом и возобновлением после остановки."
),
api_route="trudvsem/vacancies",
),

View File

@@ -16,6 +16,7 @@ from datetime import datetime
from decimal import Decimal
from pathlib import Path
from apps.core.models import BackgroundJob, JobStatus
from apps.core.services import BackgroundJobService
from apps.core.tasks import PeriodicTask as CorePeriodicTask
from apps.parsers.checko_collection import (
@@ -73,7 +74,12 @@ from apps.parsers.source_registry import PARSER_SOURCES
from celery import shared_task
from django.conf import settings
from django.db.models import Q
from organizations.models import Organization as SourceOrganization
from organizations.models import (
Organization as SourceOrganization,
)
from organizations.models import (
OrganizationSourceRecord,
)
from organizations.services import (
normalize_organization_name as normalize_identity_name,
)
@@ -141,30 +147,9 @@ class FNSApiFetchResult:
VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION = 100
VACANCY_REGISTRY_TEXT_SEARCH_MAX_PAGES_PER_ORGANIZATION = 1
VACANCY_EMPLOYER_WORD_RE = re.compile(r"[0-9A-Za-zА-Яа-яЁё]+")
VACANCY_EMPLOYER_IGNORED_WORDS = {
"ао",
"акционерное",
"государственное",
"зао",
"индивидуальный",
"ип",
"муниципальное",
"нао",
"некоммерческая",
"оао",
"общество",
"ограниченной",
"ооо",
"ответственностью",
"пао",
"предприниматель",
"публичное",
"с",
"унитарное",
"фгуп",
}
VACANCY_REGISTRY_ORGANIZATIONS_PER_TASK = 25
VACANCY_TASK_NAME = "apps.parsers.tasks.parse_trudvsem_vacancies"
TRUDVSEM_VACANCY_SOURCE = "trudvsem"
def _resolve_lookup_limit(
@@ -3787,7 +3772,427 @@ def cleanup_stale_parser_loads(
}
@shared_task(bind=True)
def _normalize_trudvsem_vacancy_sources(
vacancy_sources: list[str] | None,
) -> list[str]:
"""Оставить единственный включённый источник вакансий."""
selected = vacancy_sources or [TRUDVSEM_VACANCY_SOURCE]
disabled = sorted(set(selected) - {TRUDVSEM_VACANCY_SOURCE})
if disabled:
raise ValueError("Поддерживается только источник trudvsem")
return [TRUDVSEM_VACANCY_SOURCE]
def _vacancy_targets_signature(targets: list[RegistryLookupTarget]) -> str:
"""Вернуть безопасный отпечаток порядка организаций для возобновления."""
digest = hashlib.sha256()
for target in targets:
digest.update(target.organization_id.encode("utf-8"))
digest.update(b"\n")
return digest.hexdigest()
def _vacancy_batch_records_count(batch_id: int) -> int:
return OrganizationSourceRecord.objects.filter(
source=TRUDVSEM_VACANCY_SOURCE,
load_batch=batch_id,
).count()
def _find_resumable_vacancy_job(
*,
exclude_task_id: str,
targets_signature: str,
targets_count: int,
):
"""Найти последний совместимый остановленный проход вакансий."""
candidates = (
BackgroundJobService.get_queryset()
.filter(
task_name=VACANCY_TASK_NAME,
status__in=[JobStatus.REVOKED, JobStatus.FAILURE],
)
.exclude(task_id=exclude_task_id)
.order_by("-updated_at")[:20]
)
for candidate in candidates:
meta = candidate.meta or {}
next_offset = int(meta.get("next_offset") or 0)
batch_id = meta.get("batch_id")
if (
meta.get("targets_signature") != targets_signature
or int(meta.get("total_organizations") or 0) != targets_count
or next_offset <= 0
or next_offset >= targets_count
or batch_id is None
):
continue
load = ParserLoadLog.objects.filter(
source=ParserLoadLog.Source.TRUDVSEM,
batch_id=batch_id,
).first()
if load is not None:
return candidate, load
return None, None
def _update_vacancy_registry_checkpoint(
*,
job,
load_log: ParserLoadLog,
batch_id: int,
processed: int,
total: int,
failed: int,
targets_signature: str,
) -> int:
"""Зафиксировать прогресс после полностью обработанной организации."""
saved_count = _vacancy_batch_records_count(batch_id)
progress = 100 if total == 0 else min(99, round(processed * 100 / total))
job.meta = {
**(job.meta or {}),
"batch_id": batch_id,
"next_offset": processed,
"total_organizations": total,
"failed_organizations": failed,
"saved_records": saved_count,
"targets_signature": targets_signature,
}
job.progress = progress
job.progress_message = (
f"Обработано организаций: {processed} из {total}; "
f"сохранено вакансий: {saved_count}; ошибок: {failed}"
)
job.save(update_fields=["meta", "progress", "progress_message", "updated_at"])
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.IN_PROGRESS,
records_count=saved_count,
error_message="",
)
return saved_count
def _finish_revoked_vacancy_registry_run(
*,
job,
load_log: ParserLoadLog,
batch_id: int,
processed: int,
total: int,
failed: int,
) -> dict:
"""Закрыть отменённый проход, сохранив уже записанный результат."""
saved_count = _vacancy_batch_records_count(batch_id)
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.SKIPPED,
records_count=saved_count,
error_message=(
f"Остановлено после {processed} из {total} организаций; "
"сохранённые записи не удалены"
),
)
return {
"batch_id": batch_id,
"saved": saved_count,
"processed_organizations": processed,
"failed_organizations": failed,
"status": "revoked",
"resumed": bool((job.meta or {}).get("resumed")),
}
@dataclass
class VacancyRegistryRunState:
"""Состояние одного возобновляемого прохода вакансий."""
task_id: str
targets: list[RegistryLookupTarget]
targets_signature: str
job: BackgroundJob
load_log: ParserLoadLog
batch_id: int
next_offset: int
failed: int
resumed: bool
@property
def total(self) -> int:
return len(self.targets)
def _prepare_vacancy_registry_run(
self,
*,
registry_organization_limit: int | None,
requested_by_id: int | None,
run_task_id: str | None,
) -> VacancyRegistryRunState:
"""Создать новый проход или восстановить совместимый checkpoint."""
task_id = run_task_id or self.request.id or str(uuid.uuid4())
targets = _active_registry_vacancy_targets(limit=registry_organization_limit)
total = len(targets)
targets_signature = _vacancy_targets_signature(targets)
resumed = False
job = BackgroundJobService.get_by_task_id_or_none(task_id)
if run_task_id is not None:
if job is None:
raise RuntimeError("Не найдена родительская задача вакансий")
meta = job.meta or {}
batch_id = int(meta["batch_id"])
load_log = ParserLoadLog.objects.get(
source=ParserLoadLog.Source.TRUDVSEM,
batch_id=batch_id,
)
if meta.get("targets_signature") != targets_signature:
meta = {
**meta,
"next_offset": 0,
"failed_organizations": 0,
"total_organizations": total,
"targets_signature": targets_signature,
}
job.meta = meta
job.save(update_fields=["meta", "updated_at"])
resumed = bool(meta.get("resumed"))
else:
resumable_job, resumable_load = _find_resumable_vacancy_job(
exclude_task_id=task_id,
targets_signature=targets_signature,
targets_count=total,
)
if resumable_job is not None and resumable_load is not None:
resumable_meta = resumable_job.meta or {}
batch_id = int(resumable_meta["batch_id"])
load_log = resumable_load
next_offset = int(resumable_meta["next_offset"])
failed = int(resumable_meta.get("failed_organizations") or 0)
resumed = True
ParserLoadLogService.update(
load_log,
status=ParserLoadLog.Status.IN_PROGRESS,
records_count=_vacancy_batch_records_count(batch_id),
error_message="",
)
else:
(
load_log,
batch_id,
) = ParserLoadLogService.create_load_log_with_next_batch_id(
source=ParserLoadLog.Source.TRUDVSEM,
status=ParserLoadLog.Status.IN_PROGRESS,
)
next_offset = 0
failed = 0
job = _get_or_create_background_job(
task_id=task_id,
task_name=VACANCY_TASK_NAME,
source=ParserLoadLog.Source.TRUDVSEM,
batch_id=batch_id,
requested_by_id=requested_by_id,
meta={
"source_key": TRUDVSEM_VACANCY_SOURCE,
"next_offset": next_offset,
"total_organizations": total,
"failed_organizations": failed,
"saved_records": _vacancy_batch_records_count(batch_id),
"targets_signature": targets_signature,
"resumed": resumed,
"resumed_from_task_id": (
resumable_job.task_id if resumable_job is not None else None
),
},
)
job.mark_started()
_update_vacancy_registry_checkpoint(
job=job,
load_log=load_log,
batch_id=batch_id,
processed=next_offset,
total=total,
failed=failed,
targets_signature=targets_signature,
)
meta = job.meta or {}
return VacancyRegistryRunState(
task_id=task_id,
targets=targets,
targets_signature=targets_signature,
job=job,
load_log=load_log,
batch_id=batch_id,
next_offset=int(meta.get("next_offset") or 0),
failed=int(meta.get("failed_organizations") or 0),
resumed=resumed,
)
def _vacancy_registry_run_is_revoked(state: VacancyRegistryRunState) -> bool:
state.job.refresh_from_db(fields=["status", "meta"])
return state.job.status == JobStatus.REVOKED
def _revoked_vacancy_registry_result(state: VacancyRegistryRunState) -> dict:
return _finish_revoked_vacancy_registry_run(
job=state.job,
load_log=state.load_log,
batch_id=state.batch_id,
processed=state.next_offset,
total=state.total,
failed=state.failed,
)
def _process_vacancy_registry_chunk(
state: VacancyRegistryRunState,
*,
limit: int,
proxies: list[str] | None,
) -> bool:
"""Обработать ограниченный блок; вернуть True при отмене."""
chunk_end = min(
state.total,
state.next_offset + max(1, VACANCY_REGISTRY_ORGANIZATIONS_PER_TASK),
)
with VacanciesClient(
proxies=proxies,
sources=[TRUDVSEM_VACANCY_SOURCE],
) as client:
for index in range(state.next_offset, chunk_end):
if _vacancy_registry_run_is_revoked(state):
return True
target = state.targets[index]
try:
records = _fetch_registry_target_vacancy_records(
client,
target,
page_size=max(1, limit),
)
if records:
GenericParserRecordService.save_records(
records,
batch_id=state.batch_id,
source=ParserLoadLog.Source.TRUDVSEM,
)
except Exception as exc:
state.failed += 1
logger.warning(
"Vacancy fetch failed for registry target %d of %d: %s",
index + 1,
state.total,
exc,
)
state.next_offset = index + 1
_update_vacancy_registry_checkpoint(
job=state.job,
load_log=state.load_log,
batch_id=state.batch_id,
processed=state.next_offset,
total=state.total,
failed=state.failed,
targets_signature=state.targets_signature,
)
return _vacancy_registry_run_is_revoked(state)
def _queue_or_finish_vacancy_registry_run(
state: VacancyRegistryRunState,
*,
limit: int,
registry_organization_limit: int | None,
proxies: list[str] | None,
requested_by_id: int | None,
) -> dict:
"""Передать следующий блок в очередь или закрыть проход."""
saved_count = _vacancy_batch_records_count(state.batch_id)
if state.next_offset < state.total:
try:
parse_trudvsem_vacancies.apply_async(
kwargs={
"limit": limit,
"vacancy_sources": [TRUDVSEM_VACANCY_SOURCE],
"registry_organizations_only": True,
"registry_organization_limit": registry_organization_limit,
"proxies": proxies,
"requested_by_id": requested_by_id,
"_vacancy_run_task_id": state.task_id,
}
)
except Exception as exc:
ParserLoadLogService.mark_failed(state.load_log, str(exc))
state.job.fail(error=str(exc))
raise
return {
"batch_id": state.batch_id,
"saved": saved_count,
"processed_organizations": state.next_offset,
"failed_organizations": state.failed,
"status": "in_progress",
"resumed": state.resumed,
}
result = {
"batch_id": state.batch_id,
"saved": saved_count,
"processed_organizations": state.next_offset,
"failed_organizations": state.failed,
"status": "success",
"resumed": state.resumed,
}
if state.total > 0 and state.failed >= state.total:
message = "Не удалось обработать ни одной организации в Работа России"
result["status"] = "failure"
ParserLoadLogService.mark_failed(state.load_log, message)
state.job.fail(error=message)
return result
ParserLoadLogService.update(
state.load_log,
status=ParserLoadLog.Status.SUCCESS,
records_count=saved_count,
error_message="",
)
state.job.complete(result=result)
return result
def _run_incremental_registry_vacancies(
self,
*,
limit: int,
registry_organization_limit: int | None,
proxies: list[str] | None,
requested_by_id: int | None,
run_task_id: str | None,
) -> dict:
"""Обрабатывать реестр короткими возобновляемыми Celery-проходами."""
state = _prepare_vacancy_registry_run(
self,
registry_organization_limit=registry_organization_limit,
requested_by_id=requested_by_id,
run_task_id=run_task_id,
)
if _vacancy_registry_run_is_revoked(state):
return _revoked_vacancy_registry_result(state)
if _process_vacancy_registry_chunk(state, limit=limit, proxies=proxies):
return _revoked_vacancy_registry_result(state)
return _queue_or_finish_vacancy_registry_run(
state,
limit=limit,
registry_organization_limit=registry_organization_limit,
proxies=proxies,
requested_by_id=requested_by_id,
)
@shared_task(bind=True, acks_late=True, reject_on_worker_lost=True)
def parse_trudvsem_vacancies(
self,
*,
@@ -3801,14 +4206,50 @@ def parse_trudvsem_vacancies(
registry_organization_limit: int | None = None,
proxies: list[str] | None = None,
requested_by_id: int | None = None,
_vacancy_run_task_id: str | None = None,
) -> dict:
"""Парсинг вакансий по активным организациям реестров."""
vacancy_sources = _normalize_trudvsem_vacancy_sources(vacancy_sources)
proxies = _resolve_proxies(proxies)
if _should_fetch_registry_organization_vacancies(
registry_organizations_only=registry_organizations_only,
region_code=region_code,
company_inn=company_inn,
text=text,
):
try:
return _run_incremental_registry_vacancies(
self,
limit=limit,
registry_organization_limit=registry_organization_limit,
proxies=proxies,
requested_by_id=requested_by_id,
run_task_id=_vacancy_run_task_id,
)
except Exception as exc:
task_id = _vacancy_run_task_id or self.request.id
job = (
BackgroundJobService.get_by_task_id_or_none(task_id)
if task_id
else None
)
if job is not None and not job.is_finished:
batch_id = (job.meta or {}).get("batch_id")
if batch_id is not None:
load_log = ParserLoadLog.objects.filter(
source=ParserLoadLog.Source.TRUDVSEM,
batch_id=batch_id,
).first()
if load_log is not None:
ParserLoadLogService.mark_failed(load_log, str(exc))
job.fail(error=str(exc))
raise
return _run_generic_parser(
self,
source_key="trudvsem",
source=ParserLoadLog.Source.TRUDVSEM,
task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
task_name=VACANCY_TASK_NAME,
requested_by_id=requested_by_id,
fetch_records=lambda: _fetch_vacancy_records(
proxies=proxies,
@@ -3818,8 +4259,6 @@ def parse_trudvsem_vacancies(
company_inn=company_inn,
text=text,
vacancy_sources=vacancy_sources,
registry_organizations_only=registry_organizations_only,
registry_organization_limit=registry_organization_limit,
),
)
@@ -3833,26 +4272,9 @@ def _fetch_vacancy_records(
company_inn: str | None,
text: str | None,
vacancy_sources: list[str] | None,
registry_organizations_only: bool,
registry_organization_limit: int | None,
) -> list[GenericParserItem]:
if _should_fetch_registry_organization_vacancies(
registry_organizations_only=registry_organizations_only,
region_code=region_code,
company_inn=company_inn,
text=text,
):
return _fetch_registry_organization_vacancy_records(
proxies=proxies,
limit=limit,
vacancy_sources=vacancy_sources,
registry_organization_limit=registry_organization_limit,
)
with VacanciesClient(
proxies=proxies,
superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""),
hh_user_agent=getattr(settings, "HH_USER_AGENT", ""),
sources=vacancy_sources,
) as client:
return client.fetch_vacancies(
@@ -3914,60 +4336,11 @@ def _fetch_registry_target_vacancy_records(
*,
page_size: int,
) -> list[GenericParserItem]:
iter_source_clients = getattr(client, "iter_source_clients", None)
if iter_source_clients is None:
return _fetch_registry_target_source_vacancy_records(
client,
target,
page_size=page_size,
company_inn=target.inn,
)
records: list[GenericParserItem] = []
errors: list[str] = []
attempts = 0
for source, source_client in iter_source_clients():
if getattr(source_client, "supports_company_inn", False):
kwargs = {"company_inn": target.inn}
else:
if not target.name:
logger.info(
"Vacancy source %s is skipped for registry organization %s: "
"empty organization name",
source,
target.organization_id,
)
continue
kwargs = {"text": _vacancy_registry_text_query(target)}
attempts += 1
try:
source_records = _fetch_registry_target_source_vacancy_records(
source_client,
target,
page_size=page_size,
**kwargs,
)
except Exception as exc:
logger.warning(
"Vacancy source %s failed for registry organization %s (%s): %s",
source,
target.organization_id,
target.inn,
exc,
)
errors.append(f"{source}: {exc}")
continue
records.extend(source_records)
if errors and not records and attempts:
raise RuntimeError(
"All vacancy sources failed for registry organization "
f"{target.organization_id} ({target.inn}); first error: {errors[0]}"
)
return records
return _fetch_registry_target_source_vacancy_records(
client,
target,
page_size=page_size,
)
def _fetch_registry_target_source_vacancy_records(
@@ -3975,155 +4348,28 @@ def _fetch_registry_target_source_vacancy_records(
target: RegistryLookupTarget,
*,
page_size: int,
company_inn: str | None = None,
text: str | None = None,
) -> list[GenericParserItem]:
records: list[GenericParserItem] = []
offset = 0
filter_by_employer_name = company_inn is None
max_pages = (
VACANCY_REGISTRY_TEXT_SEARCH_MAX_PAGES_PER_ORGANIZATION
if filter_by_employer_name
else VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION
)
for _ in range(max_pages):
for _ in range(VACANCY_REGISTRY_MAX_PAGES_PER_ORGANIZATION):
page_records = source_client.fetch_vacancies(
limit=page_size,
offset=offset,
company_inn=company_inn,
text=text,
company_inn=target.inn,
)
records.extend(
_attach_registry_vacancy_target(record, target) for record in page_records
)
if filter_by_employer_name:
matched_records = [
record
for record in page_records
if _vacancy_record_matches_registry_target(record, target)
]
else:
matched_records = page_records
records.extend(matched_records)
if len(page_records) < page_size:
return records
offset += page_size
if filter_by_employer_name:
return records
raise RuntimeError(
"Vacancy registry organization page limit reached "
f"for organization {target.organization_id} ({target.inn})"
)
def _vacancy_record_matches_registry_target(
record: GenericParserItem,
target: RegistryLookupTarget,
) -> bool:
target_key = _vacancy_employer_match_key(target.name)
employer_key = _vacancy_employer_match_key(_vacancy_record_employer_name(record))
if not target_key or not employer_key:
return False
if target_key == employer_key:
return True
if min(len(target_key), len(employer_key)) < 8:
return False
return target_key in employer_key or employer_key in target_key
def _vacancy_registry_text_query(target: RegistryLookupTarget) -> str:
return _vacancy_employer_match_key(target.name) or target.name
def _vacancy_record_employer_name(record: GenericParserItem) -> str:
if record.organisation_name:
return record.organisation_name
payload = record.payload if isinstance(record.payload, dict) else {}
for key in ("employer", "company"):
nested = payload.get(key)
if isinstance(nested, dict) and nested.get("name"):
return str(nested["name"])
for key in ("firm_name", "company_name", "organisation_name"):
value = payload.get(key)
if value:
return str(value)
return ""
def _vacancy_employer_match_key(name: str) -> str:
words = []
for match in VACANCY_EMPLOYER_WORD_RE.finditer(name.casefold().replace("ё", "е")):
word = match.group(0)
if word not in VACANCY_EMPLOYER_IGNORED_WORDS:
words.append(word)
return " ".join(words)
def _fetch_registry_organization_vacancy_records(
*,
proxies: list[str] | None,
limit: int,
vacancy_sources: list[str] | None,
registry_organization_limit: int | None,
) -> list[GenericParserItem]:
targets = _active_registry_vacancy_targets(limit=registry_organization_limit)
if not targets:
logger.info("No active registry organizations for vacancies sync")
return []
page_size = max(1, limit)
records: list[GenericParserItem] = []
errors: list[str] = []
successful_fetches = 0
with VacanciesClient(
proxies=proxies,
superjob_app_id=getattr(settings, "SUPERJOB_APP_ID", ""),
hh_user_agent=getattr(settings, "HH_USER_AGENT", ""),
sources=vacancy_sources,
) as client:
for target in targets:
try:
organization_records = _fetch_registry_target_vacancy_records(
client,
target,
page_size=page_size,
)
except Exception as exc:
logger.warning(
"Vacancy fetch failed for registry organization %s (%s): %s",
target.organization_id,
target.inn,
exc,
)
errors.append(f"{target.inn}: {exc}")
continue
successful_fetches += 1
records.extend(
_attach_registry_vacancy_target(record, target)
for record in organization_records
)
if errors and successful_fetches == 0:
raise RuntimeError(
"All registry organization vacancy fetches failed; "
f"first error: {errors[0]}"
)
if errors:
logger.warning(
"Vacancy registry organization sync completed with %d failed "
"organizations",
len(errors),
)
logger.info(
"Fetched %d vacancy records for %d active registry organizations",
len(records),
len(targets),
)
return records
# =============================================================================
# FNS Tasks (File Watch & Processing)
# =============================================================================