feat: add parser source dashboard and scheduling

.gitea/workflows/ci-cd.yml

@@ -2,14 +2,15 @@ name: CI/CD Pipeline
 
 on:
   push:
-    branches: [ main, develop ]
+    branches: [ main, dev ]
   pull_request:
-    branches: [ main, develop ]
+    branches: [ main, dev ]
 
 jobs:
   lint:
     name: Code Quality Checks
     runs-on: ubuntu-latest
+    timeout-minutes: 15
 
     steps:
       - name: Checkout code
@@ -21,9 +22,7 @@ jobs:
           python-version: '3.11'
 
       - name: Install uv
-        run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
+        run: python -m pip install --upgrade pip uv
 
       - name: Create virtual environment
         run: uv venv
@@ -36,40 +35,17 @@ jobs:
       - name: Run Ruff linting
         run: |
           source .venv/bin/activate
-          ruff check .
+          ruff check src
 
       - name: Run Ruff formatting check
         run: |
           source .venv/bin/activate
-          ruff format . --check
+          ruff format src --check
 
   test:
     name: Run Tests
     runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:15.10
-        env:
-          POSTGRES_DB: test_db
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
-      redis:
-        image: redis:7-alpine
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379
+    timeout-minutes: 20
 
     steps:
       - name: Checkout code
@@ -81,9 +57,7 @@ jobs:
           python-version: '3.11'
 
       - name: Install uv
-        run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-          echo "$HOME/.local/bin" >> $GITHUB_PATH
+        run: python -m pip install --upgrade pip uv
 
       - name: Create virtual environment
         run: uv venv
@@ -93,36 +67,21 @@ jobs:
           source .venv/bin/activate
           uv sync --dev
 
-      - name: Wait for services to be ready
-        run: |
-          # Wait for PostgreSQL
-          until pg_isready -h localhost -p 5432 -U postgres; do
-            echo "Waiting for PostgreSQL..."
-            sleep 2
-          done
-
-          # Wait for Redis
-          until redis-cli -h localhost -p 6379 ping; do
-            echo "Waiting for Redis..."
-            sleep 2
-          done
-
       - name: Run Django tests
         run: |
           source .venv/bin/activate
           cd src
           python manage.py test --verbosity=2
         env:
-          DJANGO_SETTINGS_MODULE: config.settings.development
-          DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
-          REDIS_URL: redis://localhost:6379/0
-          CELERY_BROKER_URL: redis://localhost:6379/0
+          DJANGO_SETTINGS_MODULE: config.settings.test
           SECRET_KEY: test-secret-key-for-ci
 
   build:
     name: Build Docker Images
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     needs: [lint, test]
+    if: github.event_name != 'pull_request'
 
     steps:
       - name: Checkout code
@@ -181,7 +140,7 @@ jobs:
     name: Push to Gitea Registry
     runs-on: ubuntu-latest
     needs: [build]
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
 
     steps:
       - name: Checkout code

pyproject.toml

@@ -106,7 +106,7 @@ packages = ["src"]
 # ==================================================================================
 [tool.pytest.ini_options]
 DJANGO_SETTINGS_MODULE = "config.settings.test"
-python_paths = ["src"]
+pythonpath = ["src"]
 testpaths = ["tests"]
 addopts = [
     "--verbose",
@@ -127,6 +127,8 @@ markers = [
     "serializers: marks tests for serializers",
     "services: marks tests for services",
     "factories: marks tests for factories",
+    "network: marks tests that require network access",
+    "e2e: marks end-to-end tests",
 ]
 
 filterwarnings = [

src/apps/parsers/clients/common/structured.py

@@ -541,8 +541,7 @@ class StructuredDataClient:
         result = []
         for row in rows:
             values = [
-                cell.get_text(" ", strip=True)
-                for cell in row.find_all(["td", "th"])
+                cell.get_text(" ", strip=True) for cell in row.find_all(["td", "th"])
             ]
             if len(values) < 8 or self._is_fas_goz_header_number_row(values):
                 continue

tests/apps/backups/test_services_views.py

@@ -10,7 +10,9 @@ from apps.backups.models import BackupExportJob
 from apps.backups.services import BackupExportService
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.models import ParserLoadLog
+from apps.registers.models import Register
 from apps.user.services import UserService
+from django.test import override_settings
 from django.urls import reverse
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -18,19 +20,25 @@ from rest_framework.test import APITestCase
 from tests.apps.parsers.factories import GenericParserRecordFactory
 from tests.apps.registers.factories import (
     OrganizationFactory,
-    RegisterFactory,
     RegistryMembershipPeriodFactory,
 )
 from tests.apps.user.factories import UserFactory
 
 
+@override_settings(
+    BACKUP_ENCRYPTION_KEY="MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=",
+    CELERY_TASK_ALWAYS_EAGER=True,
+    CELERY_TASK_EAGER_PROPAGATES=True,
+)
 class BackupExportTest(APITestCase):
     """Tests for registry backup service and API."""
 
     def setUp(self):
         self.admin = UserFactory.create_user(is_staff=True)
         self.user = UserFactory.create_user()
-        self.registry = RegisterFactory(name="Реестр предприятий ОПК")
+        self.registry, _ = Register.objects.get_or_create(
+            name="Реестр предприятий ОПК"
+        )
         self.organization = OrganizationFactory(
             pn_name='АО "ОПК"',
             mn_ogrn=1027600980990,

tests/apps/exchange/test_views.py

@@ -132,8 +132,15 @@ class ExchangeApiTest(APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(list_response.status_code, status.HTTP_200_OK)
-        self.assertEqual(PeriodicTask.objects.count(), 1)
-        payload = list_response.data["data"][0]["payload"]
+        self.assertTrue(
+            PeriodicTask.objects.filter(id=response.data["data"]["id"]).exists()
+        )
+        created_task = next(
+            item
+            for item in list_response.data["data"]
+            if item["id"] == response.data["data"]["id"]
+        )
+        payload = created_task["payload"]
         self.assertEqual(payload["mode"], "single")
         self.assertEqual(payload["table"], "registers_organization")
 

tests/apps/parsers/test_tasks.py

@@ -7,6 +7,7 @@ from unittest.mock import patch
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.clients.common.schemas import GenericParserItem
 from apps.parsers.models import GenericParserRecord, ParserLoadLog
+from apps.parsers.source_registry import PARSER_SOURCES
 from apps.parsers.tasks import (
     import_parser_upload,
     parse_all_sources,
@@ -73,7 +74,7 @@ class GenericParserTasksTest(TestCase):
         self.assertEqual(result["status"], "success")
         self.assertEqual(
             mock_fetch_records.call_args.kwargs["file_url"],
-            "https://bo.nalog.gov.ru/advanced-search/organizations/search",
+            PARSER_SOURCES["fns_financial"].upstream_url,
         )
 
     def test_import_parser_upload_saves_records_and_removes_file(self):