Merge remote-tracking branch 'origin/main' into dev

.gitea/workflows/deploy-customer-main.yml

@@ -57,8 +57,8 @@ jobs:
         run: |
           set -euo pipefail
 
-          registry_user="${REGISTRY_USER:-${REGISTRY_USERNAME:-${GITHUB_ACTOR:-}}}"
+          registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
-          registry_password="${REGISTRY_TOKEN:-${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}}"
+          registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
           home_dir="${HOME:-/root}"
 
           if [ -z "${registry_user}" ]; then
@@ -98,8 +98,8 @@ jobs:
         run: |
           set -euo pipefail
 
-          registry_user="${REGISTRY_USER:-${REGISTRY_USERNAME:-${GITHUB_ACTOR:-}}}"
+          registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
-          registry_password="${REGISTRY_TOKEN:-${REGISTRY_PASSWORD:-${GITEA_TOKEN:-}}}"
+          registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
           sha_short="$(printf '%s' "${GITHUB_SHA}" | cut -c1-12)"
           registry_path="${CUSTOMER_REGISTRY_HOST}/${CUSTOMER_REGISTRY_NAMESPACE}"
           web_ref="${registry_path}/${CUSTOMER_WEB_IMAGE}"
@@ -132,25 +132,42 @@ jobs:
 
       - name: Deploy customer stack
         env:
+          REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
+          REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
+          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+          REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+          GITEA_TOKEN: ${{ gitea.token }}
           CUSTOMER_DEPLOY_SSH_KEY: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY }}
           CUSTOMER_DEPLOY_SSH_KEY_B64: ${{ secrets.CUSTOMER_DEPLOY_SSH_KEY_B64 }}
           DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
         run: |
           set -euo pipefail
 
+          registry_user="${REGISTRY_USERNAME:-${REGISTRY_USER:-${GITHUB_ACTOR:-}}}"
+          registry_password="${REGISTRY_PASSWORD:-${REGISTRY_TOKEN:-${GITEA_TOKEN:-}}}"
+          case "${registry_user}" in
+            *[!A-Za-z0-9._@-]*)
+              echo "Registry user contains unsupported characters" >&2
+              exit 1
+              ;;
+          esac
+
           home_dir="${HOME:-/root}"
           mkdir -p "${home_dir}/.ssh"
           key_path="${home_dir}/.ssh/customer_deploy_key"
-          if [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then
+          if [ -f "/root/.ssh/ci-key" ]; then
+            cp "/root/.ssh/ci-key" "${key_path}"
+          elif [ -f "${home_dir}/.ssh/ci-key" ]; then
+            cp "${home_dir}/.ssh/ci-key" "${key_path}"
+          elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY_B64:-}" ]; then
             printf '%s' "${CUSTOMER_DEPLOY_SSH_KEY_B64}" | base64 -d > "${key_path}"
           elif [ -n "${DEPLOY_SSH_KEY:-}" ]; then
             printf '%s' "${DEPLOY_SSH_KEY}" | base64 -d > "${key_path}"
           elif [ -n "${CUSTOMER_DEPLOY_SSH_KEY:-}" ]; then
             printf '%s\n' "${CUSTOMER_DEPLOY_SSH_KEY}" > "${key_path}"
-          elif [ -f "${home_dir}/.ssh/ci-key" ]; then
-            cp "${home_dir}/.ssh/ci-key" "${key_path}"
           else
-            cp "/root/.ssh/ci-key" "${key_path}"
+            echo "Customer deploy SSH key is unavailable" >&2
+            exit 1
           fi
           chmod 600 "${key_path}"
 
@@ -169,4 +186,7 @@ jobs:
              flock -w 1800 /tmp/ecos-customer-deploy.lock /bin/sh -c 'cd /ecos && FORCE_PULL=1 COMPOSE_FILE=\"${CUSTOMER_COMPOSE_FILE}\" \"${CUSTOMER_DEPLOY_SCRIPT}\" && docker image prune -f'"
 
           ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "true"
+          printf '%s' "${registry_password}" \
+            | ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" \
+                "docker login '${CUSTOMER_REGISTRY_HOST}' -u '${registry_user}' --password-stdin"
           ssh "${ssh_common[@]}" -o "ProxyCommand=${proxy_command}" "${CUSTOMER_DEPLOY_USER}@${CUSTOMER_DEPLOY_HOST}" "${remote_command}"

README.md

@@ -131,6 +131,18 @@ uv sync --dev
 
 Можно через `make services-up` или любым удобным способом.
 
+База State Corp должна быть создана в кодировке `UTF8`. Если шаблоны внешнего
+PostgreSQL-кластера используют `SQL_ASCII`, кодировку нужно задать явно:
+
+```bash
+createdb --owner=state_corp_app --encoding=UTF8 \
+  --locale=C.utf8 --template=template0 state_corp
+```
+
+Контейнеры web/Celery/migrate проверяют кодировку при запуске и не стартуют на
+не-UTF8 базе, чтобы ограничения `varchar` не зависели от байтового представления
+кириллицы.
+
 ### 3. Экспорт переменных
 
 ```bash

src/apps/core/excel.py

@@ -363,6 +363,8 @@ class BaseExcelParser(ABC, Generic[T]):
 
         self._workbook = openpyxl.load_workbook(content, read_only=True, data_only=True)
         self._sheet = self._workbook.active
+        if self._sheet.max_row is None or self._sheet.max_column is None:
+            self._sheet.calculate_dimension(force=True)
 
     def _parse_row(self, row_num: int) -> RowData | None:
         """Парсит одну строку Excel."""

src/apps/core/startup_checks.py

@@ -41,6 +41,15 @@ def _check_db(timeout_seconds: int) -> tuple[bool, str]:
         with conn.cursor() as cursor:
             cursor.execute("SELECT 1")
             cursor.fetchone()
+            cursor.execute("SHOW server_encoding")
+            encoding = str(cursor.fetchone()[0]).upper()
+            if encoding != "UTF8":
+                target = f"{params['host']}:{params['port']}/{params['dbname']}"
+                return (
+                    False,
+                    f"{target} (database encoding {encoding} is unsupported; "
+                    "UTF8 is required)",
+                )
         return True, "OK"
     except Exception as exc:  # noqa: BLE001
         target = f"{params['host']}:{params['port']}/{params['dbname']}"

src/apps/exchange/services.py

@@ -5,6 +5,7 @@ from __future__ import annotations
 import base64
 import hashlib
 import json
+import logging
 import struct
 import uuid
 import zlib
@@ -41,6 +42,8 @@ from django.db import transaction
 from django.db.models import Q
 from django.utils import timezone
 
+logger = logging.getLogger(__name__)
+
 
 class ExchangeImportError(ValueError):
     """Exchange package validation or import error."""
@@ -282,6 +285,13 @@ class ExchangePackageImportService:
             )
             raise
         except Exception as exc:  # noqa: BLE001
+            logger.exception(
+                "Unexpected exchange package import failure "
+                "(package_id=%s, package_hash=%s, delivery_channel=%s)",
+                package_id,
+                decoded.package_hash,
+                delivery_channel,
+            )
             error_message = "Не удалось импортировать пакет обмена"
             cls._create_failed_record(
                 decoded=decoded,

src/apps/external_data/migrations/0005_public_procurement_purchase_name_text.py

@@ -0,0 +1,15 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("external_data", "0004_auto_20260527_1928"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="publicprocurement",
+            name="purchase_name",
+            field=models.TextField(verbose_name="предмет закупки"),
+        ),
+    ]

src/apps/external_data/models.py

@@ -134,7 +134,7 @@ class PublicProcurement(UUIDPrimaryKeyMixin, TimestampMixin, models.Model):
     execution_end_date = models.DateField(
         _("дата окончания исполнения"), null=True, blank=True
     )
-    purchase_name = models.CharField(_("предмет закупки"), max_length=500)
+    purchase_name = models.TextField(_("предмет закупки"))
 
     class Meta:
         ordering = ["-contract_date", "purchase_number"]

tests/apps/core/test_excel.py

@@ -1,6 +1,9 @@
 """Tests for core excel parser."""
 
+import re
+from io import BytesIO
 from unittest.mock import MagicMock
+from zipfile import ZIP_DEFLATED, ZipFile
 
 from apps.core.excel import (
     BaseExcelParser,
@@ -15,6 +18,7 @@ from apps.core.excel import (
     validate_okpo,
 )
 from django.test import TestCase
+from openpyxl import Workbook
 
 
 class ValidatorsTest(TestCase):
@@ -168,3 +172,48 @@ class BaseExcelParserTest(TestCase):
 
         self.assertEqual(len(mappings), 2)
         self.assertEqual(mappings[0].field_name, "inn")
+
+    def test_parse_recovers_missing_read_only_worksheet_dimension(self):
+        workbook = Workbook()
+        sheet = workbook.active
+        sheet.append(["Организация", "ОКПО", "ОГРН", "ИНН"])
+        sheet.append(
+            ["Тестовая организация", "12345678", "1234567890123", "1234567890"]
+        )
+        source = BytesIO()
+        workbook.save(source)
+        workbook.close()
+
+        source.seek(0)
+        without_dimension = BytesIO()
+        with ZipFile(source, "r") as input_archive, ZipFile(
+            without_dimension,
+            "w",
+            compression=ZIP_DEFLATED,
+        ) as output_archive:
+            for item in input_archive.infolist():
+                content = input_archive.read(item.filename)
+                if item.filename == "xl/worksheets/sheet1.xml":
+                    content = re.sub(rb"<dimension[^>]*/>", b"", content, count=1)
+                output_archive.writestr(item, content)
+        without_dimension.seek(0)
+
+        created_rows = []
+
+        class TestParser(BaseExcelParser):
+            def get_column_mappings(self):
+                return []
+
+            def get_next_batch_id(self) -> int:
+                return 1
+
+            def create_record(self, row_data, batch_id):
+                created_rows.append((row_data, batch_id))
+                return MagicMock()
+
+        result = TestParser().parse(without_dimension)
+
+        self.assertEqual(result.loaded_count, 1)
+        self.assertEqual(result.skipped_count, 0)
+        self.assertEqual(created_rows[0][0].organization_name, "Тестовая организация")
+        self.assertEqual(created_rows[0][1], 1)

tests/apps/core/test_startup_checks.py

@@ -0,0 +1,38 @@
+"""Tests for fail-fast dependency checks."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from apps.core.startup_checks import _check_db
+from django.test import SimpleTestCase
+
+
+class DatabaseStartupCheckTest(SimpleTestCase):
+    """Require a Unicode-safe PostgreSQL database before startup."""
+
+    @staticmethod
+    def _connection_with_encoding(encoding: str) -> MagicMock:
+        connection = MagicMock()
+        cursor = connection.cursor.return_value.__enter__.return_value
+        cursor.fetchone.side_effect = [(1,), (encoding,)]
+        return connection
+
+    @patch("apps.core.startup_checks.psycopg2.connect")
+    def test_check_db_accepts_utf8_database(self, connect):
+        connect.return_value = self._connection_with_encoding("UTF8")
+
+        ok, message = _check_db(timeout_seconds=3)
+
+        self.assertTrue(ok)
+        self.assertEqual(message, "OK")
+
+    @patch("apps.core.startup_checks.psycopg2.connect")
+    def test_check_db_rejects_non_utf8_database(self, connect):
+        connect.return_value = self._connection_with_encoding("SQL_ASCII")
+
+        ok, message = _check_db(timeout_seconds=3)
+
+        self.assertFalse(ok)
+        self.assertIn("database encoding SQL_ASCII is unsupported", message)
+        self.assertIn("UTF8 is required", message)

tests/apps/exchange/test_api.py

@@ -11,6 +11,7 @@ import zlib
 from datetime import date
 from decimal import Decimal
 from io import BytesIO
+from unittest.mock import patch
 from zipfile import ZIP_DEFLATED, ZipFile
 
 from apps.exchange.models import ExchangeDeliveryChannel, ExchangePackageImport
@@ -35,6 +36,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 from django.conf import settings
 from django.core.files.uploadedfile import SimpleUploadedFile
 from django.core.management import call_command
+from django.db import models
 from django.urls import reverse
 from django.utils.crypto import get_random_string
 from rest_framework import status
@@ -531,6 +533,67 @@ class ExchangePackageApiTest(APITestCase):
         self.assertEqual(ExchangePackageImport.objects.count(), 0)
         self.assertEqual(Organization.objects.count(), 0)
 
+    def test_upload_logs_unexpected_import_error_with_package_context(self):
+        archive = build_exchange_archive(
+            package_id="pkg-unexpected-import-error",
+            data=build_exchange_payload(),
+        )
+
+        with patch.object(
+            ExchangePackageImportService,
+            "_import_payload",
+            side_effect=RuntimeError("database failure"),
+        ), patch("apps.exchange.services.logger.exception") as log_exception:
+            response = self.client.post(
+                self.url,
+                {"file": archive},
+                format="multipart",
+                HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN,
+            )
+
+        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+        self.assertEqual(
+            response.data,
+            {"file": ["Не удалось импортировать пакет обмена"]},
+        )
+        package_import = ExchangePackageImport.objects.get(
+            package_id="pkg-unexpected-import-error"
+        )
+        log_exception.assert_called_once_with(
+            "Unexpected exchange package import failure "
+            "(package_id=%s, package_hash=%s, delivery_channel=%s)",
+            "pkg-unexpected-import-error",
+            package_import.package_hash,
+            ExchangeDeliveryChannel.API,
+        )
+
+    def test_upload_preserves_long_public_procurement_name(self):
+        payload = build_exchange_payload()
+        long_purchase_name = ("Поставка специализированного оборудования " * 16).strip()
+        payload["public_procurements"][0]["purchase_name"] = long_purchase_name
+        archive = build_exchange_archive(
+            package_id="pkg-long-public-procurement-name",
+            data=payload,
+        )
+
+        response = self.client.post(
+            self.url,
+            {"file": archive},
+            format="multipart",
+            HTTP_X_EXCHANGE_TOKEN=TEST_TOKEN,
+        )
+
+        self.assertGreater(len(long_purchase_name), 500)
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        self.assertEqual(
+            PublicProcurement.objects.get(purchase_number="purchase-001").purchase_name,
+            long_purchase_name,
+        )
+        self.assertIsInstance(
+            PublicProcurement._meta.get_field("purchase_name"),
+            models.TextField,
+        )
+
     def test_upload_rejects_legacy_schema_version(self):
         archive = build_exchange_archive(
             package_id="pkg-legacy-schema-version",