feat: add parser source dashboard and scheduling

.gitea/workflows/ci-cd.yml

@@ -2,15 +2,14 @@ name: CI/CD Pipeline
 
 on:
   push:
-    branches: [ main, dev ]
+    branches: [ main, develop ]
   pull_request:
-    branches: [ main, dev ]
+    branches: [ main, develop ]
 
 jobs:
   lint:
     name: Code Quality Checks
     runs-on: ubuntu-latest
-    timeout-minutes: 15
 
     steps:
       - name: Checkout code
@@ -22,7 +21,9 @@ jobs:
           python-version: '3.11'
 
       - name: Install uv
-        run: python -m pip install --upgrade pip uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Create virtual environment
         run: uv venv
@@ -35,17 +36,40 @@ jobs:
       - name: Run Ruff linting
         run: |
           source .venv/bin/activate
-          ruff check src
+          ruff check .
 
       - name: Run Ruff formatting check
         run: |
           source .venv/bin/activate
-          ruff format src --check
+          ruff format . --check
 
   test:
     name: Run Tests
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    services:
+      postgres:
+        image: postgres:15.10
+        env:
+          POSTGRES_DB: test_db
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
 
     steps:
       - name: Checkout code
@@ -57,7 +81,9 @@ jobs:
           python-version: '3.11'
 
       - name: Install uv
-        run: python -m pip install --upgrade pip uv
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Create virtual environment
         run: uv venv
@@ -67,21 +93,36 @@ jobs:
           source .venv/bin/activate
           uv sync --dev
 
+      - name: Wait for services to be ready
+        run: |
+          # Wait for PostgreSQL
+          until pg_isready -h localhost -p 5432 -U postgres; do
+            echo "Waiting for PostgreSQL..."
+            sleep 2
+          done
+
+          # Wait for Redis
+          until redis-cli -h localhost -p 6379 ping; do
+            echo "Waiting for Redis..."
+            sleep 2
+          done
+
       - name: Run Django tests
         run: |
           source .venv/bin/activate
           cd src
           python manage.py test --verbosity=2
         env:
-          DJANGO_SETTINGS_MODULE: config.settings.test
+          DJANGO_SETTINGS_MODULE: config.settings.development
+          DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
+          REDIS_URL: redis://localhost:6379/0
+          CELERY_BROKER_URL: redis://localhost:6379/0
           SECRET_KEY: test-secret-key-for-ci
 
   build:
     name: Build Docker Images
     runs-on: ubuntu-latest
-    timeout-minutes: 20
     needs: [lint, test]
-    if: github.event_name != 'pull_request'
 
     steps:
       - name: Checkout code
@@ -140,7 +181,7 @@ jobs:
     name: Push to Gitea Registry
     runs-on: ubuntu-latest
     needs: [build]
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'
 
     steps:
       - name: Checkout code

pyproject.toml

@@ -106,7 +106,7 @@ packages = ["src"]
 # ==================================================================================
 [tool.pytest.ini_options]
 DJANGO_SETTINGS_MODULE = "config.settings.test"
-pythonpath = ["src"]
+python_paths = ["src"]
 testpaths = ["tests"]
 addopts = [
     "--verbose",
@@ -127,8 +127,6 @@ markers = [
     "serializers: marks tests for serializers",
     "services: marks tests for services",
     "factories: marks tests for factories",
-    "network: marks tests that require network access",
-    "e2e: marks end-to-end tests",
 ]
 
 filterwarnings = [

src/apps/parsers/clients/common/structured.py

@@ -541,7 +541,8 @@ class StructuredDataClient:
         result = []
         for row in rows:
             values = [
-                cell.get_text(" ", strip=True) for cell in row.find_all(["td", "th"])
+                cell.get_text(" ", strip=True)
+                for cell in row.find_all(["td", "th"])
             ]
             if len(values) < 8 or self._is_fas_goz_header_number_row(values):
                 continue

tests/apps/backups/test_services_views.py

@@ -10,9 +10,7 @@ from apps.backups.models import BackupExportJob
 from apps.backups.services import BackupExportService
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.models import ParserLoadLog
-from apps.registers.models import Register
 from apps.user.services import UserService
-from django.test import override_settings
 from django.urls import reverse
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -20,25 +18,19 @@ from rest_framework.test import APITestCase
 from tests.apps.parsers.factories import GenericParserRecordFactory
 from tests.apps.registers.factories import (
     OrganizationFactory,
+    RegisterFactory,
     RegistryMembershipPeriodFactory,
 )
 from tests.apps.user.factories import UserFactory
 
 
-@override_settings(
-    BACKUP_ENCRYPTION_KEY="MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=",
-    CELERY_TASK_ALWAYS_EAGER=True,
-    CELERY_TASK_EAGER_PROPAGATES=True,
-)
 class BackupExportTest(APITestCase):
     """Tests for registry backup service and API."""
 
     def setUp(self):
         self.admin = UserFactory.create_user(is_staff=True)
         self.user = UserFactory.create_user()
-        self.registry, _ = Register.objects.get_or_create(
+        self.registry = RegisterFactory(name="Реестр предприятий ОПК")
-            name="Реестр предприятий ОПК"
-        )
         self.organization = OrganizationFactory(
             pn_name='АО "ОПК"',
             mn_ogrn=1027600980990,

tests/apps/exchange/test_views.py

@@ -132,15 +132,8 @@ class ExchangeApiTest(APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(list_response.status_code, status.HTTP_200_OK)
-        self.assertTrue(
+        self.assertEqual(PeriodicTask.objects.count(), 1)
-            PeriodicTask.objects.filter(id=response.data["data"]["id"]).exists()
+        payload = list_response.data["data"][0]["payload"]
-        )
-        created_task = next(
-            item
-            for item in list_response.data["data"]
-            if item["id"] == response.data["data"]["id"]
-        )
-        payload = created_task["payload"]
         self.assertEqual(payload["mode"], "single")
         self.assertEqual(payload["table"], "registers_organization")
 

tests/apps/parsers/test_tasks.py

@@ -7,7 +7,6 @@ from unittest.mock import patch
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.clients.common.schemas import GenericParserItem
 from apps.parsers.models import GenericParserRecord, ParserLoadLog
-from apps.parsers.source_registry import PARSER_SOURCES
 from apps.parsers.tasks import (
     import_parser_upload,
     parse_all_sources,
@@ -74,7 +73,7 @@ class GenericParserTasksTest(TestCase):
         self.assertEqual(result["status"], "success")
         self.assertEqual(
             mock_fetch_records.call_args.kwargs["file_url"],
-            PARSER_SOURCES["fns_financial"].upstream_url,
+            "https://bo.nalog.gov.ru/advanced-search/organizations/search",
         )
 
     def test_import_parser_upload_saves_records_and_removes_file(self):