fix(parsers): make source jobs resumable
All checks were successful
All checks were successful
This commit is contained in:
@@ -2,9 +2,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from io import BytesIO
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.parsers.gosedo import (
|
||||
GosedoNotModified,
|
||||
GosedoParseResult,
|
||||
@@ -22,6 +24,9 @@ from apps.parsers.media_news import (
|
||||
)
|
||||
from apps.parsers.models import ParserLoadLog, ParserSourceArtifact, ParserStagedRecord
|
||||
from apps.parsers.source_artifacts import cleanup_parser_source_artifacts
|
||||
from apps.parsers.tasks import parse_media_news
|
||||
from django.core.files.base import ContentFile
|
||||
from django.core.files.storage import default_storage
|
||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||
from django.test import TestCase
|
||||
from django.urls import reverse
|
||||
@@ -519,6 +524,55 @@ class MediaNewsImportTest(TestCase):
|
||||
)
|
||||
|
||||
|
||||
class MediaNewsTaskTest(TestCase):
|
||||
def test_uploaded_workbook_task_completes_background_job(self):
|
||||
Organization.objects.create(
|
||||
name="АО Задача СМИ",
|
||||
inn="0012345678",
|
||||
ogrn="1027700132195",
|
||||
okpo="00123456",
|
||||
directory_imported_at=timezone.now(),
|
||||
)
|
||||
workbook = _media_workbook(
|
||||
[
|
||||
[
|
||||
"00123456",
|
||||
"0012345678",
|
||||
"2026-07-01",
|
||||
"СМИ",
|
||||
"https://example.test/news",
|
||||
"Источник\nЗаголовок\nЛид\nСтрока 4\nПолный текст",
|
||||
"Положительная",
|
||||
]
|
||||
]
|
||||
)
|
||||
|
||||
with TemporaryDirectory() as media_root, self.settings(MEDIA_ROOT=media_root):
|
||||
file_path = default_storage.save(
|
||||
"parser_uploads/media.xlsx",
|
||||
ContentFile(workbook.getvalue()),
|
||||
)
|
||||
task_result = parse_media_news.apply(
|
||||
kwargs={
|
||||
"file_path": file_path,
|
||||
"original_name": "media.xlsx",
|
||||
},
|
||||
task_id="media-news-task-test",
|
||||
)
|
||||
|
||||
self.assertTrue(task_result.successful())
|
||||
self.assertFalse(default_storage.exists(file_path))
|
||||
|
||||
job = BackgroundJob.objects.get(task_id="media-news-task-test")
|
||||
self.assertEqual(job.status, JobStatus.SUCCESS)
|
||||
self.assertEqual(job.progress, 100)
|
||||
self.assertEqual(task_result.result["published"], 1)
|
||||
self.assertEqual(
|
||||
ParserLoadLog.objects.get(source=ParserLoadLog.Source.MEDIA_NEWS).status,
|
||||
ParserLoadLog.Status.SUCCESS,
|
||||
)
|
||||
|
||||
|
||||
class MediaNewsPermissionsTest(APITestCase):
|
||||
def setUp(self):
|
||||
self.user = UserFactory.create_user()
|
||||
@@ -536,8 +590,11 @@ class MediaNewsPermissionsTest(APITestCase):
|
||||
|
||||
self.client.force_authenticate(self.admin)
|
||||
with patch(
|
||||
"apps.parsers.views._save_uploaded_parser_file",
|
||||
return_value="parser_uploads/media.xlsx",
|
||||
), patch(
|
||||
"apps.parsers.tasks.parse_media_news.apply_async",
|
||||
return_value=SimpleNamespace(id="media-task-1"),
|
||||
side_effect=lambda **kwargs: SimpleNamespace(id=kwargs["task_id"]),
|
||||
):
|
||||
response = self.client.post(
|
||||
self.url,
|
||||
@@ -546,8 +603,21 @@ class MediaNewsPermissionsTest(APITestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
|
||||
self.assertEqual(response.data["data"]["task_id"], "media-task-1")
|
||||
self.assertEqual(response.data["data"]["task_ids"], ["media-task-1"])
|
||||
task_id = response.data["data"]["task_id"]
|
||||
self.assertEqual(response.data["data"]["task_ids"], [task_id])
|
||||
|
||||
card_response = self.client.get(
|
||||
reverse(
|
||||
"api_v1:sources:source-cards-detail",
|
||||
kwargs={"slug": "media-mentions"},
|
||||
)
|
||||
)
|
||||
self.assertEqual(card_response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(card_response.data["data"]["status"], "in_progress")
|
||||
self.assertEqual(
|
||||
card_response.data["data"]["active_tasks"][0]["task_id"],
|
||||
task_id,
|
||||
)
|
||||
|
||||
def test_gosedo_manual_run_is_admin_only_and_returns_task_ids(self):
|
||||
url = reverse(
|
||||
|
||||
@@ -441,6 +441,31 @@ class SourceCardServiceDatabaseTest(TestCase):
|
||||
def setUp(self):
|
||||
SourceCardService.clear_cache()
|
||||
|
||||
def test_enqueue_vacancy_refresh_reuses_fresh_active_job(self):
|
||||
existing = BackgroundJob.objects.create(
|
||||
task_id="active-vacancies",
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
|
||||
status=JobStatus.STARTED,
|
||||
)
|
||||
task = MagicMock()
|
||||
|
||||
result = SourceCardService._enqueue_task(
|
||||
task=task,
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies",
|
||||
requested_by_id=5,
|
||||
meta={"source_card": "labor-vacancies"},
|
||||
kwargs={"requested_by_id": 5},
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
result,
|
||||
{
|
||||
"task_id": existing.task_id,
|
||||
"task_name": existing.task_name,
|
||||
},
|
||||
)
|
||||
task.apply_async.assert_not_called()
|
||||
|
||||
def test_defense_unreliable_suppliers_counts_unique_generic_organizations(self):
|
||||
_save_source_record(
|
||||
source=ParserLoadLog.Source.UNFAIR_SUPPLIERS,
|
||||
|
||||
@@ -2,6 +2,7 @@ from dataclasses import asdict
|
||||
|
||||
from apps.parsers import tasks
|
||||
from apps.parsers.clients.common.structured import MAX_FILE_SIZE_BYTES
|
||||
from apps.parsers.serializers import ParserRunRequestSerializer
|
||||
from apps.parsers.source_registry import PARSER_SOURCES
|
||||
from apps.parsers.views import TASKS_BY_NAME
|
||||
from django.conf import settings
|
||||
@@ -53,3 +54,19 @@ class ParserSourceRegistryPresentationTest(SimpleTestCase):
|
||||
)
|
||||
|
||||
self.assertNotIn("checko", public_metadata.lower())
|
||||
|
||||
def test_vacancy_source_metadata_names_only_trudvsem(self):
|
||||
source = PARSER_SOURCES["trudvsem"]
|
||||
|
||||
self.assertEqual(source.agency, "Работа России")
|
||||
self.assertEqual(source.parser_strategy, "incremental_trudvsem_api")
|
||||
self.assertNotIn("HH", source.data_scope)
|
||||
self.assertNotIn("SuperJob", source.data_scope)
|
||||
|
||||
def test_vacancy_run_request_rejects_disabled_sources(self):
|
||||
serializer = ParserRunRequestSerializer(
|
||||
data={"vacancy_sources": ["hh", "superjob"]}
|
||||
)
|
||||
|
||||
self.assertFalse(serializer.is_valid())
|
||||
self.assertIn("vacancy_sources", serializer.errors)
|
||||
|
||||
@@ -16,6 +16,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.core.services import BackgroundJobService
|
||||
from apps.parsers import tasks as parser_tasks
|
||||
from apps.parsers.clients.base import HTTPError
|
||||
@@ -37,7 +38,11 @@ from apps.parsers.models import (
|
||||
FinancialReport,
|
||||
ParserLoadLog,
|
||||
)
|
||||
from apps.parsers.services import FNSReportService, ParserLoadLogService
|
||||
from apps.parsers.services import (
|
||||
FNSReportService,
|
||||
GenericParserRecordService,
|
||||
ParserLoadLogService,
|
||||
)
|
||||
from apps.parsers.tasks import (
|
||||
FNS_API_DEFAULT_REGISTRY_ORGANIZATION_LIMIT,
|
||||
FNS_API_MAX_REGISTRY_ORGANIZATION_LIMIT,
|
||||
@@ -2869,6 +2874,19 @@ class TaskHelpersTestCase(TestCase):
|
||||
|
||||
|
||||
class ParseVacanciesTaskTestCase(TestCase):
|
||||
def _make_registry_organization(self, *, inn: int, name: str) -> Organization:
|
||||
organization = OrganizationFactory(
|
||||
pn_name=name,
|
||||
mn_inn=inn,
|
||||
mn_ogrn=inn + 1000000000000,
|
||||
)
|
||||
RegistryMembershipPeriodFactory(organization=organization, ended_at=None)
|
||||
return organization
|
||||
|
||||
def test_vacancy_task_requeues_after_worker_loss(self):
|
||||
self.assertTrue(parse_trudvsem_vacancies.acks_late)
|
||||
self.assertTrue(parse_trudvsem_vacancies.reject_on_worker_lost)
|
||||
|
||||
def test_parse_trudvsem_vacancies_without_filters_fetches_active_registry_orgs(
|
||||
self,
|
||||
):
|
||||
@@ -3052,170 +3070,28 @@ class ParseVacanciesTaskTestCase(TestCase):
|
||||
{"trudvsem:7701000102"},
|
||||
)
|
||||
|
||||
@override_settings(SUPERJOB_APP_ID="test-superjob-app-id")
|
||||
def test_parse_trudvsem_vacancies_matches_job_boards_by_employer_name(self):
|
||||
organization = OrganizationFactory(
|
||||
pn_name='Общество с ограниченной ответственностью "Ромашка"',
|
||||
mn_inn=7701000301,
|
||||
mn_ogrn=1027700000301,
|
||||
)
|
||||
RegistryMembershipPeriodFactory(organization=organization, ended_at=None)
|
||||
captured_client_kwargs = {}
|
||||
captured_text_queries = {}
|
||||
def test_parse_trudvsem_vacancies_rejects_disabled_sources(self):
|
||||
_ensure_directory_organization(inn="7701000401")
|
||||
|
||||
class _Provider:
|
||||
def __init__(self, source_name, *, supports_company_inn):
|
||||
self.source_name = source_name
|
||||
self.supports_company_inn = supports_company_inn
|
||||
with self.assertRaisesMessage(
|
||||
ValueError,
|
||||
"Поддерживается только источник trudvsem",
|
||||
):
|
||||
parse_trudvsem_vacancies(
|
||||
limit=25,
|
||||
region_code="1",
|
||||
text="инженер",
|
||||
proxies=[],
|
||||
vacancy_sources=["hh", "superjob"],
|
||||
)
|
||||
|
||||
def fetch_vacancies(self, **kwargs):
|
||||
if self.source_name == "trudvsem":
|
||||
return [
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id="trudvsem:romashka",
|
||||
inn=kwargs["company_inn"],
|
||||
title="Работа России",
|
||||
payload={"vacancy_source": "trudvsem"},
|
||||
)
|
||||
]
|
||||
self.assertFalse(OrganizationSourceRecord.objects.exists())
|
||||
|
||||
captured_text_queries[self.source_name] = kwargs["text"]
|
||||
return [
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id=f"{self.source_name}:romashka",
|
||||
organisation_name='ООО "Ромашка"',
|
||||
title=f"{self.source_name} matching vacancy",
|
||||
payload={"vacancy_source": self.source_name},
|
||||
),
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id=f"{self.source_name}:other",
|
||||
organisation_name='ООО "Лютик"',
|
||||
title=f"{self.source_name} unrelated vacancy",
|
||||
payload={"vacancy_source": self.source_name},
|
||||
),
|
||||
]
|
||||
|
||||
class _VacanciesClient:
|
||||
def __init__(self, **kwargs):
|
||||
captured_client_kwargs.update(kwargs)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
def fetch_vacancies(self, **kwargs):
|
||||
return [
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id="trudvsem:romashka",
|
||||
inn=kwargs["company_inn"],
|
||||
title="Работа России",
|
||||
payload={"vacancy_source": "trudvsem"},
|
||||
)
|
||||
]
|
||||
|
||||
def iter_source_clients(self):
|
||||
return [
|
||||
("trudvsem", _Provider("trudvsem", supports_company_inn=True)),
|
||||
("hh", _Provider("hh", supports_company_inn=False)),
|
||||
("superjob", _Provider("superjob", supports_company_inn=False)),
|
||||
]
|
||||
|
||||
original_client = parser_tasks.VacanciesClient
|
||||
parser_tasks.VacanciesClient = _VacanciesClient
|
||||
try:
|
||||
result = parse_trudvsem_vacancies(limit=50, proxies=[])
|
||||
finally:
|
||||
parser_tasks.VacanciesClient = original_client
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertIsNone(captured_client_kwargs["sources"])
|
||||
self.assertEqual(
|
||||
captured_text_queries,
|
||||
{
|
||||
"hh": "ромашка",
|
||||
"superjob": "ромашка",
|
||||
},
|
||||
)
|
||||
self.assertEqual(result["saved"], 3)
|
||||
self.assertEqual(
|
||||
{
|
||||
(
|
||||
record.external_id,
|
||||
record.source,
|
||||
record.registry_organization_id,
|
||||
)
|
||||
for record in OrganizationSourceRecord.objects.all()
|
||||
},
|
||||
{
|
||||
("trudvsem:romashka", "trudvsem", organization.id),
|
||||
("hh:romashka", "hh", organization.id),
|
||||
("superjob:romashka", "superjob", organization.id),
|
||||
},
|
||||
)
|
||||
|
||||
def test_registry_job_board_matching_fetches_only_first_text_search_page(self):
|
||||
organization = OrganizationFactory(
|
||||
pn_name='Общество с ограниченной ответственностью "Ромашка"',
|
||||
mn_inn=7701000302,
|
||||
mn_ogrn=1027700000302,
|
||||
)
|
||||
RegistryMembershipPeriodFactory(organization=organization, ended_at=None)
|
||||
captured_offsets = []
|
||||
|
||||
class _Provider:
|
||||
supports_company_inn = False
|
||||
|
||||
def fetch_vacancies(self, **kwargs):
|
||||
captured_offsets.append(kwargs["offset"])
|
||||
return [
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id=f"hh:romashka:{kwargs['offset']}",
|
||||
organisation_name='ООО "Ромашка"',
|
||||
title="HeadHunter",
|
||||
payload={"vacancy_source": "hh"},
|
||||
)
|
||||
]
|
||||
|
||||
class _VacanciesClient:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
def iter_source_clients(self):
|
||||
return [("hh", _Provider())]
|
||||
|
||||
original_client = parser_tasks.VacanciesClient
|
||||
parser_tasks.VacanciesClient = _VacanciesClient
|
||||
try:
|
||||
result = parse_trudvsem_vacancies(limit=1, proxies=[])
|
||||
finally:
|
||||
parser_tasks.VacanciesClient = original_client
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["saved"], 1)
|
||||
self.assertEqual(captured_offsets, [0])
|
||||
|
||||
@override_settings(
|
||||
SUPERJOB_APP_ID="test-superjob-app-id",
|
||||
HH_USER_AGENT="Mostovik/1.0 (ops@example.test)",
|
||||
)
|
||||
def test_parse_trudvsem_vacancies_uses_combined_vacancies_client(self):
|
||||
for inn in ("7701000401", "7701000402", "7701000403"):
|
||||
_ensure_directory_organization(inn=inn)
|
||||
def test_registry_run_persists_a_chunk_and_queues_continuation(self):
|
||||
self._make_registry_organization(inn=7701000501, name="АО Первый пакет")
|
||||
self._make_registry_organization(inn=7701000502, name="АО Второй пакет")
|
||||
captured_kwargs = {}
|
||||
captured_fetch_kwargs = {}
|
||||
captured_fetches = []
|
||||
|
||||
class _VacanciesClient:
|
||||
def __init__(self, **kwargs):
|
||||
@@ -3228,70 +3104,175 @@ class ParseVacanciesTaskTestCase(TestCase):
|
||||
return None
|
||||
|
||||
def fetch_vacancies(self, **kwargs):
|
||||
captured_fetch_kwargs.update(kwargs)
|
||||
captured_fetches.append(kwargs["company_inn"])
|
||||
return [
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id="trudvsem:1",
|
||||
inn="7701000401",
|
||||
external_id=f"trudvsem:{kwargs['company_inn']}",
|
||||
inn=kwargs["company_inn"],
|
||||
title="Работа России",
|
||||
payload={"vacancy_source": "trudvsem"},
|
||||
),
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id="hh:1",
|
||||
inn="7701000402",
|
||||
title="HeadHunter",
|
||||
payload={"vacancy_source": "hh"},
|
||||
),
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id="superjob:1",
|
||||
inn="7701000403",
|
||||
title="SuperJob",
|
||||
payload={"vacancy_source": "superjob"},
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
original_client = parser_tasks.VacanciesClient
|
||||
parser_tasks.VacanciesClient = _VacanciesClient
|
||||
try:
|
||||
result = parse_trudvsem_vacancies(
|
||||
limit=25,
|
||||
offset=5,
|
||||
region_code="1",
|
||||
text="инженер",
|
||||
proxies=[],
|
||||
vacancy_sources=["trudvsem", "hh", "superjob"],
|
||||
)
|
||||
with patch.object(
|
||||
parser_tasks,
|
||||
"VACANCY_REGISTRY_ORGANIZATIONS_PER_TASK",
|
||||
1,
|
||||
create=True,
|
||||
), patch.object(
|
||||
parser_tasks.parse_trudvsem_vacancies,
|
||||
"apply_async",
|
||||
return_value=SimpleNamespace(id="vacancy-continuation"),
|
||||
) as apply_async_mock:
|
||||
first_result = parse_trudvsem_vacancies(limit=25, proxies=[])
|
||||
continuation_kwargs = apply_async_mock.call_args.kwargs["kwargs"]
|
||||
|
||||
self.assertEqual(first_result["status"], "in_progress")
|
||||
self.assertEqual(captured_fetches, ["7701000501"])
|
||||
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
|
||||
|
||||
job = BackgroundJob.objects.get(
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies"
|
||||
)
|
||||
load = ParserLoadLog.objects.get(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=job.meta["batch_id"],
|
||||
)
|
||||
self.assertEqual(job.status, JobStatus.STARTED)
|
||||
self.assertEqual(job.progress, 50)
|
||||
self.assertEqual(job.meta["next_offset"], 1)
|
||||
self.assertEqual(load.status, ParserLoadLog.Status.IN_PROGRESS)
|
||||
self.assertEqual(load.records_count, 1)
|
||||
|
||||
final_result = parse_trudvsem_vacancies(**continuation_kwargs)
|
||||
finally:
|
||||
parser_tasks.VacanciesClient = original_client
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["saved"], 3)
|
||||
self.assertEqual(captured_kwargs["superjob_app_id"], "test-superjob-app-id")
|
||||
self.assertEqual(
|
||||
captured_kwargs["hh_user_agent"],
|
||||
"Mostovik/1.0 (ops@example.test)",
|
||||
self.assertEqual(final_result["status"], "success")
|
||||
self.assertEqual(final_result["saved"], 2)
|
||||
self.assertEqual(captured_fetches, ["7701000501", "7701000502"])
|
||||
self.assertEqual(captured_kwargs["sources"], ["trudvsem"])
|
||||
self.assertEqual(OrganizationSourceRecord.objects.count(), 2)
|
||||
job.refresh_from_db()
|
||||
load.refresh_from_db()
|
||||
self.assertEqual(job.status, JobStatus.SUCCESS)
|
||||
self.assertEqual(job.progress, 100)
|
||||
self.assertEqual(load.status, ParserLoadLog.Status.SUCCESS)
|
||||
self.assertEqual(load.records_count, 2)
|
||||
|
||||
def test_registry_run_preserves_and_resumes_records_after_revoke(self):
|
||||
self._make_registry_organization(inn=7701000601, name="АО До остановки")
|
||||
self._make_registry_organization(inn=7701000602, name="АО После остановки")
|
||||
captured_fetches = []
|
||||
|
||||
class _VacanciesClient:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
def fetch_vacancies(self, **kwargs):
|
||||
captured_fetches.append(kwargs["company_inn"])
|
||||
return [
|
||||
GenericParserItem(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
external_id=f"trudvsem:{kwargs['company_inn']}",
|
||||
inn=kwargs["company_inn"],
|
||||
title="Работа России",
|
||||
payload={"vacancy_source": "trudvsem"},
|
||||
)
|
||||
]
|
||||
|
||||
original_client = parser_tasks.VacanciesClient
|
||||
original_save_records = GenericParserRecordService.save_records
|
||||
|
||||
def save_then_revoke(records, batch_id, *, source, chunk_size=500):
|
||||
saved = original_save_records(
|
||||
records,
|
||||
batch_id,
|
||||
source=source,
|
||||
chunk_size=chunk_size,
|
||||
)
|
||||
job = BackgroundJob.objects.get(
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies"
|
||||
)
|
||||
job.revoke()
|
||||
return saved
|
||||
|
||||
parser_tasks.VacanciesClient = _VacanciesClient
|
||||
try:
|
||||
with patch.object(
|
||||
GenericParserRecordService,
|
||||
"save_records",
|
||||
side_effect=save_then_revoke,
|
||||
):
|
||||
stopped_result = parse_trudvsem_vacancies(limit=25, proxies=[])
|
||||
|
||||
stopped_job = BackgroundJob.objects.get(
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies"
|
||||
)
|
||||
stopped_load = ParserLoadLog.objects.get(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=stopped_job.meta["batch_id"],
|
||||
)
|
||||
self.assertEqual(stopped_result["status"], "revoked")
|
||||
self.assertEqual(stopped_job.meta["next_offset"], 1)
|
||||
self.assertEqual(stopped_load.status, ParserLoadLog.Status.SKIPPED)
|
||||
self.assertEqual(stopped_load.records_count, 1)
|
||||
self.assertEqual(OrganizationSourceRecord.objects.count(), 1)
|
||||
|
||||
resumed_result = parse_trudvsem_vacancies(limit=25, proxies=[])
|
||||
finally:
|
||||
parser_tasks.VacanciesClient = original_client
|
||||
|
||||
self.assertEqual(resumed_result["status"], "success")
|
||||
self.assertEqual(resumed_result["batch_id"], stopped_result["batch_id"])
|
||||
self.assertTrue(resumed_result["resumed"])
|
||||
self.assertEqual(captured_fetches, ["7701000601", "7701000602"])
|
||||
self.assertEqual(OrganizationSourceRecord.objects.count(), 2)
|
||||
|
||||
def test_registry_run_fails_when_trudvsem_fails_for_every_organization(self):
|
||||
self._make_registry_organization(inn=7701000701, name="АО Ошибка источника")
|
||||
|
||||
class _VacanciesClient:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
def fetch_vacancies(self, **kwargs):
|
||||
raise RuntimeError("external 503")
|
||||
|
||||
original_client = parser_tasks.VacanciesClient
|
||||
parser_tasks.VacanciesClient = _VacanciesClient
|
||||
try:
|
||||
result = parse_trudvsem_vacancies(limit=25, proxies=[])
|
||||
finally:
|
||||
parser_tasks.VacanciesClient = original_client
|
||||
|
||||
job = BackgroundJob.objects.get(
|
||||
task_name="apps.parsers.tasks.parse_trudvsem_vacancies"
|
||||
)
|
||||
self.assertEqual(captured_kwargs["sources"], ["trudvsem", "hh", "superjob"])
|
||||
self.assertEqual(captured_fetch_kwargs["limit"], 25)
|
||||
self.assertEqual(captured_fetch_kwargs["offset"], 5)
|
||||
self.assertEqual(captured_fetch_kwargs["region_code"], "1")
|
||||
self.assertEqual(captured_fetch_kwargs["text"], "инженер")
|
||||
self.assertEqual(
|
||||
set(
|
||||
OrganizationSourceRecord.objects.values_list(
|
||||
"external_id",
|
||||
"source",
|
||||
)
|
||||
),
|
||||
{
|
||||
("trudvsem:1", "trudvsem"),
|
||||
("hh:1", "hh"),
|
||||
("superjob:1", "superjob"),
|
||||
},
|
||||
load = ParserLoadLog.objects.get(
|
||||
source=ParserLoadLog.Source.TRUDVSEM,
|
||||
batch_id=job.meta["batch_id"],
|
||||
)
|
||||
self.assertEqual(result["status"], "failure")
|
||||
self.assertEqual(job.status, JobStatus.FAILURE)
|
||||
self.assertEqual(load.status, ParserLoadLog.Status.FAILED)
|
||||
self.assertFalse(OrganizationSourceRecord.objects.exists())
|
||||
|
||||
|
||||
class ParserLoadLogServiceTestCase(TestCase):
|
||||
|
||||
@@ -196,7 +196,7 @@ class SuperJobVacanciesClientTest(SimpleTestCase):
|
||||
|
||||
|
||||
class VacanciesClientTest(SimpleTestCase):
|
||||
def test_fetch_vacancies_combines_enabled_sources(self):
|
||||
def test_fetch_vacancies_uses_only_trudvsem_by_default(self):
|
||||
class _Provider:
|
||||
def __init__(self, source_name: str):
|
||||
self.source_name = source_name
|
||||
@@ -222,7 +222,7 @@ class VacanciesClientTest(SimpleTestCase):
|
||||
|
||||
self.assertEqual(
|
||||
[record.payload["vacancy_source"] for record in records],
|
||||
["trudvsem", "hh", "superjob"],
|
||||
["trudvsem"],
|
||||
)
|
||||
|
||||
def test_fetch_vacancies_skips_sources_without_company_inn_support(self):
|
||||
@@ -278,8 +278,11 @@ class VacanciesClientTest(SimpleTestCase):
|
||||
|
||||
self.assertEqual([record.external_id for record in records], ["trudvsem:1"])
|
||||
|
||||
def test_fetch_vacancies_requires_superjob_app_id_when_explicitly_selected(self):
|
||||
client = VacanciesClient(sources=["superjob"], superjob_app_id="")
|
||||
def test_fetch_vacancies_rejects_disabled_sources(self):
|
||||
client = VacanciesClient(sources=["hh", "superjob"])
|
||||
|
||||
with self.assertRaises(VacanciesClientError):
|
||||
with self.assertRaisesMessage(
|
||||
VacanciesClientError,
|
||||
"Disabled vacancy sources: hh, superjob",
|
||||
):
|
||||
client.fetch_vacancies()
|
||||
|
||||
Reference in New Issue
Block a user