feat: add parser source dashboard and scheduling

.gitea/workflows/ci-cd.yml

@@ -2,9 +2,9 @@ name: CI/CD Pipeline
 
 on:
   push:
-    branches: [ main, dev ]
+    branches: [ main, develop ]
   pull_request:
-    branches: [ main, dev ]
+    branches: [ main, develop ]
 
 jobs:
   lint:
@@ -113,7 +113,7 @@ jobs:
           cd src
           python manage.py test --verbosity=2
         env:
-          DJANGO_SETTINGS_MODULE: config.settings.test
+          DJANGO_SETTINGS_MODULE: config.settings.development
           DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
           REDIS_URL: redis://localhost:6379/0
           CELERY_BROKER_URL: redis://localhost:6379/0
@@ -181,7 +181,7 @@ jobs:
     name: Push to Gitea Registry
     runs-on: ubuntu-latest
     needs: [build]
-    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev'
+    if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop'
 
     steps:
       - name: Checkout code

src/apps/parsers/clients/common/structured.py

@@ -541,7 +541,8 @@ class StructuredDataClient:
         result = []
         for row in rows:
             values = [
-                cell.get_text(" ", strip=True) for cell in row.find_all(["td", "th"])
+                cell.get_text(" ", strip=True)
+                for cell in row.find_all(["td", "th"])
             ]
             if len(values) < 8 or self._is_fas_goz_header_number_row(values):
                 continue

tests/apps/backups/test_services_views.py

@@ -10,9 +10,7 @@ from apps.backups.models import BackupExportJob
 from apps.backups.services import BackupExportService
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.models import ParserLoadLog
-from apps.registers.models import Register
 from apps.user.services import UserService
-from django.test import override_settings
 from django.urls import reverse
 from rest_framework import status
 from rest_framework.test import APITestCase
@@ -20,25 +18,19 @@ from rest_framework.test import APITestCase
 from tests.apps.parsers.factories import GenericParserRecordFactory
 from tests.apps.registers.factories import (
     OrganizationFactory,
+    RegisterFactory,
     RegistryMembershipPeriodFactory,
 )
 from tests.apps.user.factories import UserFactory
 
 
-@override_settings(
-    BACKUP_ENCRYPTION_KEY="MDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDA=",
-    CELERY_TASK_ALWAYS_EAGER=True,
-    CELERY_TASK_EAGER_PROPAGATES=True,
-)
 class BackupExportTest(APITestCase):
     """Tests for registry backup service and API."""
 
     def setUp(self):
         self.admin = UserFactory.create_user(is_staff=True)
         self.user = UserFactory.create_user()
-        self.registry, _ = Register.objects.get_or_create(
-            name="Реестр предприятий ОПК"
-        )
+        self.registry = RegisterFactory(name="Реестр предприятий ОПК")
         self.organization = OrganizationFactory(
             pn_name='АО "ОПК"',
             mn_ogrn=1027600980990,

tests/apps/exchange/test_views.py

@@ -132,15 +132,8 @@ class ExchangeApiTest(APITestCase):
 
         self.assertEqual(response.status_code, status.HTTP_201_CREATED)
         self.assertEqual(list_response.status_code, status.HTTP_200_OK)
-        self.assertTrue(
-            PeriodicTask.objects.filter(id=response.data["data"]["id"]).exists()
-        )
-        created_task = next(
-            item
-            for item in list_response.data["data"]
-            if item["id"] == response.data["data"]["id"]
-        )
-        payload = created_task["payload"]
+        self.assertEqual(PeriodicTask.objects.count(), 1)
+        payload = list_response.data["data"][0]["payload"]
         self.assertEqual(payload["mode"], "single")
         self.assertEqual(payload["table"], "registers_organization")
 

tests/apps/parsers/test_tasks.py

@@ -7,7 +7,6 @@ from unittest.mock import patch
 from apps.core.models import BackgroundJob, JobStatus
 from apps.parsers.clients.common.schemas import GenericParserItem
 from apps.parsers.models import GenericParserRecord, ParserLoadLog
-from apps.parsers.source_registry import PARSER_SOURCES
 from apps.parsers.tasks import (
     import_parser_upload,
     parse_all_sources,
@@ -74,7 +73,7 @@ class GenericParserTasksTest(TestCase):
         self.assertEqual(result["status"], "success")
         self.assertEqual(
             mock_fetch_records.call_args.kwargs["file_url"],
-            PARSER_SOURCES["fns_financial"].upstream_url,
+            "https://bo.nalog.gov.ru/advanced-search/organizations/search",
         )
 
     def test_import_parser_upload_saves_records_and_removes_file(self):