fix(parsers): unblock source status and heavy product import
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
@@ -22,6 +23,7 @@ from apps.parsers.clients.minpromtorg.schemas import (
|
||||
from apps.parsers.clients.proverki import ProverkiClient
|
||||
from apps.parsers.clients.proverki.schemas import Inspection
|
||||
from django.test import TestCase, tag
|
||||
from openpyxl import load_workbook as openpyxl_load_workbook
|
||||
from requests.adapters import BaseAdapter
|
||||
|
||||
from tests.utils import Response, TestHTTPServer
|
||||
@@ -513,6 +515,40 @@ class IndustrialProductsClientTest(TestCase):
|
||||
|
||||
self.assertEqual(products, [])
|
||||
|
||||
def test_fetch_products_uses_read_only_workbook(self):
|
||||
excel_bytes, rows = build_minpromtorg_products_excel(count=2)
|
||||
file_name = "industrial_products_20260428.xlsx"
|
||||
|
||||
with TestHTTPServer() as server:
|
||||
server.add_json(
|
||||
"/api/kss-document-preview",
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"name": IndustrialProductsClient().query,
|
||||
"files": [
|
||||
{"name": file_name, "url": f"/files/{file_name}"}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
server.add_bytes(f"/files/{file_name}", excel_bytes)
|
||||
|
||||
client = IndustrialProductsClient(
|
||||
host=_host_from_base_url(server.base_url),
|
||||
scheme="http",
|
||||
http_adapter=server.adapter,
|
||||
)
|
||||
with patch(
|
||||
"apps.parsers.clients.minpromtorg.products.load_workbook",
|
||||
wraps=openpyxl_load_workbook,
|
||||
) as load_workbook_mock:
|
||||
products = client.fetch_products()
|
||||
|
||||
self.assertEqual(len(products), len(rows))
|
||||
self.assertTrue(load_workbook_mock.call_args.kwargs["read_only"])
|
||||
|
||||
def test_get_latest_file_url_falls_back_to_excel_file(self):
|
||||
client = IndustrialProductsClient()
|
||||
files = [
|
||||
|
||||
@@ -5,6 +5,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from apps.core.models import BackgroundJob, JobStatus
|
||||
from apps.parsers.models import GenericParserRecord, ParserLoadLog
|
||||
from apps.parsers.source_cards import (
|
||||
SourceCardDefinition,
|
||||
SourceCardService,
|
||||
@@ -296,6 +297,42 @@ class SourceCardServiceUnitTest(SimpleTestCase):
|
||||
|
||||
@override_settings(PARSER_STALE_LOAD_MAX_AGE_MINUTES=90)
|
||||
class SourceCardServiceDatabaseTest(TestCase):
|
||||
def test_public_procurements_counts_generic_eis_sources(self):
|
||||
GenericParserRecord.objects.create(
|
||||
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
load_batch=1,
|
||||
external_id="notice-1",
|
||||
inn="7701234567",
|
||||
title="Закупка 44-ФЗ",
|
||||
payload={"number": "notice-1"},
|
||||
)
|
||||
GenericParserRecord.objects.create(
|
||||
source=ParserLoadLog.Source.CONTRACTS,
|
||||
load_batch=1,
|
||||
external_id="contract-1",
|
||||
inn="7701234567",
|
||||
title="Контракт ЕИС",
|
||||
payload={"number": "contract-1"},
|
||||
)
|
||||
ParserLoadLog.objects.create(
|
||||
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
|
||||
batch_id=1,
|
||||
records_count=1,
|
||||
status=ParserLoadLog.Status.SUCCESS,
|
||||
)
|
||||
ParserLoadLog.objects.create(
|
||||
source=ParserLoadLog.Source.CONTRACTS,
|
||||
batch_id=1,
|
||||
records_count=1,
|
||||
status=ParserLoadLog.Status.SUCCESS,
|
||||
)
|
||||
|
||||
card = SourceCardService.get_card("public-procurements")
|
||||
|
||||
self.assertEqual(card["status"], "success")
|
||||
self.assertEqual(card["records_count"], 2)
|
||||
self.assertEqual(card["organizations_count"], 1)
|
||||
|
||||
def test_get_active_tasks_ignores_old_jobs_even_when_updated_recently(self):
|
||||
job = BackgroundJob.objects.create(
|
||||
task_id="old-source-task",
|
||||
|
||||
@@ -40,6 +40,8 @@ from apps.parsers.models import (
|
||||
)
|
||||
from apps.parsers.services import ParserLoadLogService
|
||||
from apps.parsers.tasks import (
|
||||
INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS,
|
||||
INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS,
|
||||
_move_to_dir,
|
||||
_process_fns_file_sync,
|
||||
_remove_lock,
|
||||
@@ -589,6 +591,16 @@ class SyncProcurementsTaskTestCase(TestCase):
|
||||
class MinpromtorgTasksTestCase(TestCase):
|
||||
"""Tests for Minpromtorg tasks."""
|
||||
|
||||
def test_parse_industrial_products_has_extended_time_limits(self):
|
||||
self.assertEqual(
|
||||
parse_industrial_products.soft_time_limit,
|
||||
INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS,
|
||||
)
|
||||
self.assertEqual(
|
||||
parse_industrial_products.time_limit,
|
||||
INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS,
|
||||
)
|
||||
|
||||
def _add_minpromtorg_routes(self, server: TestHTTPServer):
|
||||
certificates_bytes, cert_rows = build_minpromtorg_certificates_excel(count=2)
|
||||
manufacturers_bytes, manuf_rows = build_minpromtorg_manufacturers_excel(count=2)
|
||||
|
||||
Reference in New Issue
Block a user