fix(parsers): unblock source status and heavy product import
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 19s
CI/CD Pipeline / Build and Push Images (push) Successful in 6s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev in Dokploy (push) Successful in 1s

This commit is contained in:
2026-04-28 22:51:52 +02:00
parent e921f4b204
commit 01387ae13b
6 changed files with 182 additions and 25 deletions

View File

@@ -2,6 +2,7 @@
from __future__ import annotations
from unittest.mock import patch
from urllib.parse import urlparse
import requests
@@ -22,6 +23,7 @@ from apps.parsers.clients.minpromtorg.schemas import (
from apps.parsers.clients.proverki import ProverkiClient
from apps.parsers.clients.proverki.schemas import Inspection
from django.test import TestCase, tag
from openpyxl import load_workbook as openpyxl_load_workbook
from requests.adapters import BaseAdapter
from tests.utils import Response, TestHTTPServer
@@ -513,6 +515,40 @@ class IndustrialProductsClientTest(TestCase):
self.assertEqual(products, [])
def test_fetch_products_uses_read_only_workbook(self):
excel_bytes, rows = build_minpromtorg_products_excel(count=2)
file_name = "industrial_products_20260428.xlsx"
with TestHTTPServer() as server:
server.add_json(
"/api/kss-document-preview",
{
"data": [
{
"name": IndustrialProductsClient().query,
"files": [
{"name": file_name, "url": f"/files/{file_name}"}
],
}
]
},
)
server.add_bytes(f"/files/{file_name}", excel_bytes)
client = IndustrialProductsClient(
host=_host_from_base_url(server.base_url),
scheme="http",
http_adapter=server.adapter,
)
with patch(
"apps.parsers.clients.minpromtorg.products.load_workbook",
wraps=openpyxl_load_workbook,
) as load_workbook_mock:
products = client.fetch_products()
self.assertEqual(len(products), len(rows))
self.assertTrue(load_workbook_mock.call_args.kwargs["read_only"])
def test_get_latest_file_url_falls_back_to_excel_file(self):
client = IndustrialProductsClient()
files = [

View File

@@ -5,6 +5,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from apps.core.models import BackgroundJob, JobStatus
from apps.parsers.models import GenericParserRecord, ParserLoadLog
from apps.parsers.source_cards import (
SourceCardDefinition,
SourceCardService,
@@ -296,6 +297,42 @@ class SourceCardServiceUnitTest(SimpleTestCase):
@override_settings(PARSER_STALE_LOAD_MAX_AGE_MINUTES=90)
class SourceCardServiceDatabaseTest(TestCase):
def test_public_procurements_counts_generic_eis_sources(self):
GenericParserRecord.objects.create(
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
load_batch=1,
external_id="notice-1",
inn="7701234567",
title="Закупка 44-ФЗ",
payload={"number": "notice-1"},
)
GenericParserRecord.objects.create(
source=ParserLoadLog.Source.CONTRACTS,
load_batch=1,
external_id="contract-1",
inn="7701234567",
title="Контракт ЕИС",
payload={"number": "contract-1"},
)
ParserLoadLog.objects.create(
source=ParserLoadLog.Source.PROCUREMENTS_44FZ,
batch_id=1,
records_count=1,
status=ParserLoadLog.Status.SUCCESS,
)
ParserLoadLog.objects.create(
source=ParserLoadLog.Source.CONTRACTS,
batch_id=1,
records_count=1,
status=ParserLoadLog.Status.SUCCESS,
)
card = SourceCardService.get_card("public-procurements")
self.assertEqual(card["status"], "success")
self.assertEqual(card["records_count"], 2)
self.assertEqual(card["organizations_count"], 1)
def test_get_active_tasks_ignores_old_jobs_even_when_updated_recently(self):
job = BackgroundJob.objects.create(
task_id="old-source-task",

View File

@@ -40,6 +40,8 @@ from apps.parsers.models import (
)
from apps.parsers.services import ParserLoadLogService
from apps.parsers.tasks import (
INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS,
INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS,
_move_to_dir,
_process_fns_file_sync,
_remove_lock,
@@ -589,6 +591,16 @@ class SyncProcurementsTaskTestCase(TestCase):
class MinpromtorgTasksTestCase(TestCase):
"""Tests for Minpromtorg tasks."""
def test_parse_industrial_products_has_extended_time_limits(self):
self.assertEqual(
parse_industrial_products.soft_time_limit,
INDUSTRIAL_PRODUCTS_SOFT_TIME_LIMIT_SECONDS,
)
self.assertEqual(
parse_industrial_products.time_limit,
INDUSTRIAL_PRODUCTS_TIME_LIMIT_SECONDS,
)
def _add_minpromtorg_routes(self, server: TestHTTPServer):
certificates_bytes, cert_rows = build_minpromtorg_certificates_excel(count=2)
manufacturers_bytes, manuf_rows = build_minpromtorg_manufacturers_excel(count=2)