avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(parsers): sync RU proxies from proxy-tools
Some checks failed
CI/CD Pipeline / Telegram Notify Success (push) Has been cancelled
Details
CI/CD Pipeline / Run Tests (push) Has been cancelled
Details
CI/CD Pipeline / Code Quality Checks (push) Has been cancelled
Details

Browse Source
This commit is contained in:
Aleksandr Meshchriakov
2026-03-23 10:47:34 +01:00
parent b4260d53cb
commit 7d4c54636b
13 changed files with 705 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/apps/core/test_celery_module.py
View File

@@ -52,6 +52,7 @@ class CeleryModuleTest(SimpleTestCase):
self.assertIn("parse-manufactures-daily", module.app.conf.beat_schedule)
self.assertIn("parse-industrial-products-daily", module.app.conf.beat_schedule)
self.assertIn("parse-inspections-weekly", module.app.conf.beat_schedule)
self.assertIn("sync-ru-proxies-hourly", module.app.conf.beat_schedule)
def test_startup_refresh_queues_when_lock_acquired(self):
with patch.dict(

2
tests/apps/parsers/factories.py
View File

@@ -83,6 +83,8 @@ class ProxyFactory(factory.django.DjangoModelFactory):
address = factory.LazyFunction(generate_proxy_address)
description = factory.LazyAttribute(lambda _: fake.sentence(nb_words=3))
source = "manual"
country_code = "RU"
is_active = True
fail_count = 0
last_used_at = factory.LazyAttribute(

125
tests/apps/parsers/test_services.py
View File

@@ -1,5 +1,6 @@
"""Tests for parsers services."""
from unittest.mock import patch
from urllib.parse import urlparse
from apps.parsers.clients.minpromtorg.industrial import IndustrialProductionClient
@@ -27,9 +28,10 @@ from apps.parsers.services import (
ParserLoadLogService,
ProcurementService,
ProxyService,
ProxyToolsSyncService,
)
from apps.registers.models import Organization
from django.test import TestCase, tag
from django.test import TestCase, override_settings, tag
from tests.utils import TestHTTPServer
from tests.utils.fixtures import build_minpromtorg_certificates_excel, fake
@@ -173,6 +175,127 @@ class ProxyServiceTest(TestCase):
self.assertEqual(created, 1)
self.assertEqual(Proxy.objects.count(), 2)
def test_get_runtime_proxies_prefers_proxy_tools_ru(self):
"""Runtime should prefer RU proxies imported from Proxy-Tools."""
manual_ru = ProxyFactory(
source=ProxyService.MANUAL_SOURCE,
country_code="RU",
)
imported_ru = ProxyFactory(
source=ProxyService.PROXY_TOOLS_SOURCE,
country_code="RU",
)
ProxyFactory(
source=ProxyService.PROXY_TOOLS_SOURCE,
country_code="US",
)
result = ProxyService.get_runtime_proxies()
self.assertEqual(result, [imported_ru.address])
self.assertNotIn(manual_ru.address, result)
def test_get_runtime_proxies_falls_back_to_any_ru_proxy(self):
"""Runtime should fall back to any RU proxy when imported list is empty."""
manual_ru = ProxyFactory(
source=ProxyService.MANUAL_SOURCE,
country_code="RU",
)
ProxyFactory(
source=ProxyService.MANUAL_SOURCE,
country_code="US",
)
result = ProxyService.get_runtime_proxies()
self.assertEqual(result, [manual_ru.address])
class ProxyToolsSyncServiceTest(TestCase):
"""Tests for ProxyToolsSyncService."""
def test_sync_ru_proxies_skips_without_api_key(self):
"""Sync should be skipped when API key is missing."""
result = ProxyToolsSyncService.sync_ru_proxies()
self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "missing_api_key")
@override_settings(
PROXY_TOOLS_API_KEY="test-token",
PROXY_TOOLS_LIMIT=2,
PROXY_TOOLS_MAX_PAGES=2,
)
@patch("apps.parsers.services.ProxyToolsClient.fetch_proxies")
def test_sync_ru_proxies_upserts_and_deactivates(self, fetch_proxies_mock):
"""Sync should create, reactivate and deactivate imported proxies."""
active_stale = ProxyFactory(
address="http://10.0.0.10:8000",
source=ProxyService.PROXY_TOOLS_SOURCE,
country_code="RU",
is_active=True,
)
inactive_existing = ProxyFactory(
address="http://10.0.0.20:8000",
source=ProxyService.PROXY_TOOLS_SOURCE,
country_code="RU",
is_active=False,
)
manual_ru = ProxyFactory(
address="http://10.0.0.30:8000",
source=ProxyService.MANUAL_SOURCE,
country_code="RU",
is_active=True,
)
fetch_proxies_mock.side_effect = [
{
"data": [
{"host": "10.0.0.20", "port": 8000, "type": "4"},
{"proxy": "socks5://10.0.0.40:1080"},
],
"meta": {"total_pages": 2},
},
{
"data": [
"https://10.0.0.50:8443",
],
"meta": {"total_pages": 2},
},
]
result = ProxyToolsSyncService.sync_ru_proxies()
self.assertEqual(result["status"], "success")
self.assertEqual(result["fetched"], 3)
self.assertEqual(result["created"], 2)
self.assertEqual(result["updated"], 1)
self.assertEqual(result["deactivated"], 1)
active_stale.refresh_from_db()
inactive_existing.refresh_from_db()
manual_ru.refresh_from_db()
self.assertFalse(active_stale.is_active)
self.assertTrue(inactive_existing.is_active)
self.assertTrue(manual_ru.is_active)
imported_addresses = set(
Proxy.objects.filter(
source=ProxyService.PROXY_TOOLS_SOURCE,
country_code="RU",
is_active=True,
).values_list("address", flat=True)
)
self.assertSetEqual(
imported_addresses,
{
"http://10.0.0.20:8000",
"socks5://10.0.0.40:1080",
"https://10.0.0.50:8443",
},
)
class ParserLoadLogServiceTest(TestCase):
"""Tests for ParserLoadLogService."""

53
tests/apps/parsers/test_tasks.py
View File

@@ -9,6 +9,7 @@ import tempfile
import threading
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
from urllib.parse import urlparse
from apps.parsers import tasks as parser_tasks
@@ -39,6 +40,7 @@ from apps.parsers.tasks import (
_move_to_dir,
_process_fns_file_sync,
_remove_lock,
_resolve_proxies,
_try_create_lock,
parse_all_minpromtorg,
parse_all_sources,
@@ -51,6 +53,7 @@ from apps.parsers.tasks import (
scan_fns_directory,
sync_inspections,
sync_procurements,
sync_ru_proxies,
)
from django.test import TestCase, override_settings
from openpyxl import Workbook
@@ -59,6 +62,7 @@ from tests.apps.parsers.factories import (
InspectionRecordFactory,
ParserLoadLogFactory,
ProcurementRecordFactory,
ProxyFactory,
)
from tests.utils import TestHTTPServer
from tests.utils.fixtures import (
@@ -102,6 +106,55 @@ def _portal_path(year: int, month: int) -> str:
return f"/portal/public-open-data/check/{year}/{month}"
class ProxyResolutionTestCase(TestCase):
"""Tests for proxy resolution in parser tasks."""
@override_settings(PARSER_PROXIES=["http://env-proxy:8080"])
def test_resolve_proxies_prefers_runtime_db_proxies(self):
imported_proxy = ProxyFactory(
address="http://10.0.0.2:8000",
source="proxy-tools",
country_code="RU",
is_active=True,
)
ProxyFactory(
address="http://10.0.0.3:8000",
source="manual",
country_code="RU",
is_active=True,
)
result = _resolve_proxies(None)
self.assertEqual(result, [imported_proxy.address])
@override_settings(PARSER_PROXIES=["http://env-proxy:8080"])
def test_resolve_proxies_falls_back_to_settings_when_db_empty(self):
result = _resolve_proxies(None)
self.assertEqual(result, ["http://env-proxy:8080"])
class SyncRuProxiesTaskTestCase(TestCase):
"""Tests for periodic RU proxy sync task."""
@patch("apps.parsers.tasks.ProxyToolsSyncService.sync_ru_proxies")
def test_sync_ru_proxies_returns_service_payload(self, sync_mock):
sync_mock.return_value = {
"status": "success",
"fetched": 3,
"created": 2,
"updated": 1,
"deactivated": 0,
}
result = sync_ru_proxies.run()
self.assertEqual(result["status"], "success")
self.assertEqual(result["fetched"], 3)
sync_mock.assert_called_once_with()
@override_settings(
CELERY_TASK_ALWAYS_EAGER=True,
CELERY_TASK_EAGER_PROPAGATES=True,