From dec81f6e7cd74f688c9467325a5772a6dd061cad Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 14:23:05 +0200 Subject: [PATCH 1/5] fix: harden state corp exchange imports --- README.md | 12 ++++ src/apps/core/startup_checks.py | 9 +++ src/apps/exchange/services.py | 10 +++ ...5_public_procurement_purchase_name_text.py | 15 +++++ src/apps/external_data/models.py | 2 +- tests/apps/core/test_startup_checks.py | 38 +++++++++++ tests/apps/exchange/test_api.py | 67 +++++++++++++++++++ 7 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/apps/external_data/migrations/0005_public_procurement_purchase_name_text.py create mode 100644 tests/apps/core/test_startup_checks.py diff --git a/README.md b/README.md index ae8f916..9bf0c12 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,18 @@ uv sync --dev Можно через `make services-up` или любым удобным способом. +База State Corp должна быть создана в кодировке `UTF8`. Если шаблоны внешнего +PostgreSQL-кластера используют `SQL_ASCII`, кодировку нужно задать явно: + +```bash +createdb --owner=state_corp_app --encoding=UTF8 \ + --locale=C.utf8 --template=template0 state_corp +``` + +Контейнеры web/Celery/migrate проверяют кодировку при запуске и не стартуют на +не-UTF8 базе, чтобы ограничения `varchar` не зависели от байтового представления +кириллицы. + ### 3. Экспорт переменных ```bash diff --git a/src/apps/core/startup_checks.py b/src/apps/core/startup_checks.py index 4053990..05c9607 100644 --- a/src/apps/core/startup_checks.py +++ b/src/apps/core/startup_checks.py @@ -41,6 +41,15 @@ def _check_db(timeout_seconds: int) -> tuple[bool, str]: with conn.cursor() as cursor: cursor.execute("SELECT 1") cursor.fetchone() + cursor.execute("SHOW server_encoding") + encoding = str(cursor.fetchone()[0]).upper() + if encoding != "UTF8": + target = f"{params['host']}:{params['port']}/{params['dbname']}" + return ( + False, + f"{target} (database encoding {encoding} is unsupported; " + "UTF8 is required)", + ) return True, "OK" except Exception as exc: # noqa: BLE001 target = f"{params['host']}:{params['port']}/{params['dbname']}" diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py index ca0773b..50bcfe9 100644 --- a/src/apps/exchange/services.py +++ b/src/apps/exchange/services.py @@ -5,6 +5,7 @@ from __future__ import annotations import base64 import hashlib import json +import logging import struct import uuid import zlib @@ -41,6 +42,8 @@ from django.db import transaction from django.db.models import Q from django.utils import timezone +logger = logging.getLogger(__name__) + class ExchangeImportError(ValueError): """Exchange package validation or import error.""" @@ -282,6 +285,13 @@ class ExchangePackageImportService: ) raise except Exception as exc: # noqa: BLE001 + logger.exception( + "Unexpected exchange package import failure " + "(package_id=%s, package_hash=%s, delivery_channel=%s)", + package_id, + decoded.package_hash, + delivery_channel, + ) error_message = "Не удалось импортировать пакет обмена" cls._create_failed_record( decoded=decoded, diff --git a/src/apps/external_data/migrations/0005_public_procurement_purchase_name_text.py b/src/apps/external_data/migrations/0005_public_procurement_purchase_name_text.py new file mode 100644 index 0000000..99b2575 --- /dev/null +++ b/src/apps/external_data/migrations/0005_public_procurement_purchase_name_text.py @@ -0,0 +1,15 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("external_data", "0004_auto_20260527_1928"), + ] + + operations = [ + migrations.AlterField( + model_name="publicprocurement", + name="purchase_name", + field=models.TextField(verbose_name="предмет закупки"), + ), + ] diff --git a/src/apps/external_data/models.py b/src/apps/external_data/models.py index c8e2c6d..3228dc7 100644 --- a/src/apps/external_data/models.py +++ b/src/apps/external_data/models.py @@ -134,7 +134,7 @@ class PublicProcurement(UUIDPrimaryKeyMixin, TimestampMixin, models.Model): execution_end_date = models.DateField( _("дата окончания исполнения"), null=True, blank=True ) - purchase_name = models.CharField(_("предмет закупки"), max_length=500) + purchase_name = models.TextField(_("предмет закупки")) class Meta: ordering = ["-contract_date", "purchase_number"] diff --git a/tests/apps/core/test_startup_checks.py b/tests/apps/core/test_startup_checks.py new file mode 100644 index 0000000..9720d9a --- /dev/null +++ b/tests/apps/core/test_startup_checks.py @@ -0,0 +1,38 @@ +"""Tests for fail-fast dependency checks.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from apps.core.startup_checks import _check_db +from django.test import SimpleTestCase + + +class DatabaseStartupCheckTest(SimpleTestCase): + """Require a Unicode-safe PostgreSQL database before startup.""" + + @staticmethod + def _connection_with_encoding(encoding: str) -> MagicMock: + connection = MagicMock() + cursor = connection.cursor.return_value.__enter__.return_value + cursor.fetchone.side_effect = [(1,), (encoding,)] + return connection + + @patch("apps.core.startup_checks.psycopg2.connect") + def test_check_db_accepts_utf8_database(self, connect): + connect.return_value = self._connection_with_encoding("UTF8") + + ok, message = _check_db(timeout_seconds=3) + + self.assertTrue(ok) + self.assertEqual(message, "OK") + + @patch("apps.core.startup_checks.psycopg2.connect") + def test_check_db_rejects_non_utf8_database(self, connect): + connect.return_value = self._connection_with_encoding("SQL_ASCII") + + ok, message = _check_db(timeout_seconds=3) + + self.assertFalse(ok) + self.assertIn("database encoding SQL_ASCII is unsupported", message) + self.assertIn("UTF8 is required", message) diff --git a/tests/apps/exchange/test_api.py b/tests/apps/exchange/test_api.py index 10366cb..1bed720 100644 --- a/tests/apps/exchange/test_api.py +++ b/tests/apps/exchange/test_api.py @@ -11,6 +11,7 @@ import zlib from datetime import date from decimal import Decimal from io import BytesIO +from unittest.mock import patch from zipfile import ZIP_DEFLATED, ZipFile from apps.exchange.models import ExchangeDeliveryChannel, ExchangePackageImport @@ -35,6 +36,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM from django.conf import settings from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management import call_command +from django.db import models from django.urls import reverse from django.utils.crypto import get_random_string from rest_framework import status @@ -531,6 +533,71 @@ class ExchangePackageApiTest(APITestCase): self.assertEqual(ExchangePackageImport.objects.count(), 0) self.assertEqual(Organization.objects.count(), 0) + def test_upload_logs_unexpected_import_error_with_package_context(self): + archive = build_exchange_archive( + package_id="pkg-unexpected-import-error", + data=build_exchange_payload(), + ) + + with patch.object( + ExchangePackageImportService, + "_import_payload", + side_effect=RuntimeError("database failure"), + ), patch("apps.exchange.services.logger.exception") as log_exception: + response = self.client.post( + self.url, + {"file": archive}, + format="multipart", + HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN, + ) + + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual( + response.data, + {"file": ["Не удалось импортировать пакет обмена"]}, + ) + package_import = ExchangePackageImport.objects.get( + package_id="pkg-unexpected-import-error" + ) + log_exception.assert_called_once_with( + "Unexpected exchange package import failure " + "(package_id=%s, package_hash=%s, delivery_channel=%s)", + "pkg-unexpected-import-error", + package_import.package_hash, + ExchangeDeliveryChannel.API, + ) + + def test_upload_preserves_long_public_procurement_name(self): + payload = build_exchange_payload() + long_purchase_name = ( + "Поставка специализированного оборудования " * 16 + ).strip() + payload["public_procurements"][0]["purchase_name"] = long_purchase_name + archive = build_exchange_archive( + package_id="pkg-long-public-procurement-name", + data=payload, + ) + + response = self.client.post( + self.url, + {"file": archive}, + format="multipart", + HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN, + ) + + self.assertGreater(len(long_purchase_name), 500) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual( + PublicProcurement.objects.get( + purchase_number="purchase-001" + ).purchase_name, + long_purchase_name, + ) + self.assertIsInstance( + PublicProcurement._meta.get_field("purchase_name"), + models.TextField, + ) + def test_upload_rejects_legacy_schema_version(self): archive = build_exchange_archive( package_id="pkg-legacy-schema-version", From db09d6e4b5a82cc18ca7de41b86ecd16a0e9cc61 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 14:28:42 +0200 Subject: [PATCH 2/5] style: format exchange import regression test --- tests/apps/exchange/test_api.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/apps/exchange/test_api.py b/tests/apps/exchange/test_api.py index 1bed720..8b5f162 100644 --- a/tests/apps/exchange/test_api.py +++ b/tests/apps/exchange/test_api.py @@ -569,9 +569,7 @@ class ExchangePackageApiTest(APITestCase): def test_upload_preserves_long_public_procurement_name(self): payload = build_exchange_payload() - long_purchase_name = ( - "Поставка специализированного оборудования " * 16 - ).strip() + long_purchase_name = ("Поставка специализированного оборудования " * 16).strip() payload["public_procurements"][0]["purchase_name"] = long_purchase_name archive = build_exchange_archive( package_id="pkg-long-public-procurement-name", @@ -588,9 +586,7 @@ class ExchangePackageApiTest(APITestCase): self.assertGreater(len(long_purchase_name), 500) self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual( - PublicProcurement.objects.get( - purchase_number="purchase-001" - ).purchase_name, + PublicProcurement.objects.get(purchase_number="purchase-001").purchase_name, long_purchase_name, ) self.assertIsInstance( From 21ae8ba50188ae7a288ffb814f34e43c114f2978 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 14:46:35 +0200 Subject: [PATCH 3/5] fix: prefer active registry credentials --- .gitea/workflows/deploy-customer-main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/deploy-customer-main.yml b/.gitea/workflows/deploy-customer-main.yml index 5fdf328..7bcd0c6 100644 --- a/.gitea/workflows/deploy-customer-main.yml +++ b/.gitea/workflows/deploy-customer-main.yml @@ -57,8 +57,8 @@ jobs: run: | set -euo pipefail - registry_user="${REGISTRY_USER:-${REGISTRY_USERNAME:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_TOKEN:-${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}}" + registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" + registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" home_dir="${HOME:-/root}" if [ -z "${registry_user}" ]; then @@ -98,8 +98,8 @@ jobs: run: | set -euo pipefail - registry_user="${REGISTRY_USER:-${REGISTRY_USERNAME:-${GITHUB_ACTOR:-}}}" - registry_password="${REGISTRY_TOKEN:-${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}}" + registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" + registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)" registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}" web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}" From b5968acedb31435b7c0d5eada1e965fa8048963c Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 15:29:57 +0200 Subject: [PATCH 4/5] fix: authenticate customer deploy transport --- .gitea/workflows/deploy-customer-main.yml | 28 +++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/deploy-customer-main.yml b/.gitea/workflows/deploy-customer-main.yml index 7bcd0c6..cdf4e3b 100644 --- a/.gitea/workflows/deploy-customer-main.yml +++ b/.gitea/workflows/deploy-customer-main.yml @@ -132,25 +132,42 @@ jobs: - name: Deploy customer stack env: + REGISTRY_USER: ${{ secrets.REGISTRY_USER }} + REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} + REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }} + REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + GITEA_TOKEN: ${{ gitea.token }} CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }} CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }} DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} run: | set -euo pipefail + registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}" + registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}" + case "${registry_user}" in + *[!A-Za-z0-9._@-]*) + echo "Registry user contains unsupported characters" >&2 + exit 1 + ;; + esac + home_dir="${HOME:-/root}" mkdir -p "${home_dir}/.ssh" key_path="${home_dir}/.ssh/customer_deploy_key" - if [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then + if [ -f "/root/.ssh/ci-key" ]; then + cp "/root/.ssh/ci-key" "${key_path}" + elif [ -f "${home_dir}/.ssh/ci-key" ]; then + cp "${home_dir}/.ssh/ci-key" "${key_path}" + elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}" elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}" elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}" - elif [ -f "${home_dir}/.ssh/ci-key" ]; then - cp "${home_dir}/.ssh/ci-key" "${key_path}" else - cp "/root/.ssh/ci-key" "${key_path}" + echo "Customer deploy SSH key is unavailable" >&2 + exit 1 fi chmod 600 "${key_path}" @@ -169,4 +186,7 @@ jobs: flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'" ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true" + printf '%s' "${registry_password}" \ + | ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \ + "docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin" ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}" From 08632be39f9c0ff4d73e1296490e169e69041c62 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Wed, 22 Jul 2026 17:20:39 +0200 Subject: [PATCH 5/5] fix: parse worksheets without dimensions --- src/apps/core/excel.py | 2 ++ tests/apps/core/test_excel.py | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/apps/core/excel.py b/src/apps/core/excel.py index 4645214..c725353 100644 --- a/src/apps/core/excel.py +++ b/src/apps/core/excel.py @@ -363,6 +363,8 @@ class BaseExcelParser(ABC, Generic[T]): self._workbook = openpyxl.load_workbook(content, read_only=True, data_only=True) self._sheet = self._workbook.active + if self._sheet.max_row is None or self._sheet.max_column is None: + self._sheet.calculate_dimension(force=True) def _parse_row(self, row_num: int) -> RowData | None: """Парсит одну строку Excel.""" diff --git a/tests/apps/core/test_excel.py b/tests/apps/core/test_excel.py index 36423ba..398e2f8 100644 --- a/tests/apps/core/test_excel.py +++ b/tests/apps/core/test_excel.py @@ -1,6 +1,9 @@ """Tests for core excel parser.""" +import re +from io import BytesIO from unittest.mock import MagicMock +from zipfile import ZIP_DEFLATED, ZipFile from apps.core.excel import ( BaseExcelParser, @@ -15,6 +18,7 @@ from apps.core.excel import ( validate_okpo, ) from django.test import TestCase +from openpyxl import Workbook class ValidatorsTest(TestCase): @@ -168,3 +172,48 @@ class BaseExcelParserTest(TestCase): self.assertEqual(len(mappings), 2) self.assertEqual(mappings[0].field_name, "inn") + + def test_parse_recovers_missing_read_only_worksheet_dimension(self): + workbook = Workbook() + sheet = workbook.active + sheet.append(["Организация", "ОКПО", "ОГРН", "ИНН"]) + sheet.append( + ["Тестовая организация", "12345678", "1234567890123", "1234567890"] + ) + source = BytesIO() + workbook.save(source) + workbook.close() + + source.seek(0) + without_dimension = BytesIO() + with ZipFile(source, "r") as input_archive, ZipFile( + without_dimension, + "w", + compression=ZIP_DEFLATED, + ) as output_archive: + for item in input_archive.infolist(): + content = input_archive.read(item.filename) + if item.filename == "xl/worksheets/sheet1.xml": + content = re.sub(rb"]*/>", b"", content, count=1) + output_archive.writestr(item, content) + without_dimension.seek(0) + + created_rows = [] + + class TestParser(BaseExcelParser): + def get_column_mappings(self): + return [] + + def get_next_batch_id(self) -> int: + return 1 + + def create_record(self, row_data, batch_id): + created_rows.append((row_data, batch_id)) + return MagicMock() + + result = TestParser().parse(without_dimension) + + self.assertEqual(result.loaded_count, 1) + self.assertEqual(result.skipped_count, 0) + self.assertEqual(created_rows[0][0].organization_name, "Тестовая организация") + self.assertEqual(created_rows[0][1], 1)