Refactor project structure and update configurations for State Corp backend

- Updated project description in __init__.py - Enhanced .gitignore to exclude additional data files - Modified User model to remove first_name and last_name fields - Improved instance save method in services.py to include updated_at field - Added API tokens to .env.example for external services - Cleaned up test files for better readability - Updated Dockerfile and docker-compose.yml for improved setup - Revised README.md to reflect project changes and added changelog
2026-02-17 09:24:42 +01:00
parent e9d7f24aaa
commit fd2adf9ab4
31 changed files with 1419 additions and 933 deletions
--- a/src/config/api_v1_urls.py
+++ b/src/config/api_v1_urls.py
@@ -1,7 +1,23 @@
 """
 API v1 URL configuration.

-All API endpoints are versioned under /api/v1/
+Все API эндпоинты версионированы под /api/v1/
+
+Структура:
+- /api/v1/users/           - Аутентификация и пользователи
+- /api/v1/jobs/            - Фоновые задачи
+- /api/v1/organizations/   - Справочник организаций
+- /api/v1/forms/f1/        - Форма Ф-1 (Выпуск продукции)
+- /api/v1/forms/f2/        - Форма Ф-2 (Бухгалтерский баланс)
+- /api/v1/forms/f3/        - Форма Ф-3 (Кадры и оборудование)
+- /api/v1/forms/f4/        - Форма Ф-4 (Сводные финансы)
+- /api/v1/forms/f5/        - Форма Ф-5 (Инвентаризация оборудования)
+- /api/v1/forms/f6/        - Форма Ф-6 (Возрастная структура)
+- /api/v1/minpromtorg/     - Минпромторг (сертификаты, производители)
+- /api/v1/proverki/        - Единый реестр проверок
+- /api/v1/zakupki/         - Государственные закупки
+- /api/v1/fns/             - ФНС (бухгалтерская отчетность)
+- /api/v1/system/          - Системные (логи, прокси) - только для админов
 """

 from apps.core.views import BackgroundJobListView, BackgroundJobStatusView
@@ -9,12 +25,24 @@ from django.urls import include, path

 app_name = "api_v1"

+# Фоновые задачи
 jobs_urlpatterns = [
    path("", BackgroundJobListView.as_view(), name="job-list"),
    path("<str:task_id>/", BackgroundJobStatusView.as_view(), name="job-status"),
 ]

 urlpatterns = [
+    # Аутентификация и пользователи
    path("users/", include("apps.user.urls")),
+    # Фоновые задачи
    path("jobs/", include((jobs_urlpatterns, "jobs"))),
+    # Справочники
+    path("organizations/", include("apps.organization.urls")),
+    # Формы отчётности
+    path("forms/f1/", include("apps.form_1.urls")),
+    path("forms/f2/", include("apps.form_2.urls")),
+    path("forms/f3/", include("apps.form_3.urls")),
+    path("forms/f4/", include("apps.form_4.urls")),
+    path("forms/f5/", include("apps.form_5.urls")),
+    path("forms/f6/", include("apps.form_6.urls")),
 ]
--- a/src/config/celery.py
+++ b/src/config/celery.py
@@ -8,8 +8,13 @@ import os

 from celery import Celery

-# Set the default Django settings module for the 'celery' program.
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development")
+# Set the Django settings module for the 'celery' program.
+if "DJANGO_SETTINGS_MODULE" not in os.environ:
+    raise RuntimeError(
+        "DJANGO_SETTINGS_MODULE is not set. "
+        "Export it explicitly before starting Celery "
+        "(e.g., config.settings.production or config.settings.development)."
+    )

 app = Celery("project")

@@ -24,17 +29,24 @@ app.autodiscover_tasks()

 # Configure Celery Beat schedule
 app.conf.beat_schedule = {
-    "check-pending-scraping-jobs": {
-        "task": "apps.scraping.tasks.check_pending_jobs",
-        "schedule": 300.0,  # Every 5 minutes
+    # Парсинг сертификатов промышленного производства - каждый день в 3:00
+    "parse-industrial-production-daily": {
+        "task": "apps.parsers.tasks.parse_industrial_production",
+        "schedule": 86400.0,  # Every 24 hours
    },
-    "process-extracted-data": {
-        "task": "apps.data_processor.tasks.process_extracted_data",
-        "schedule": 600.0,  # Every 10 minutes
+    # Парсинг реестра производителей - каждый день в 4:00
+    "parse-manufactures-daily": {
+        "task": "apps.parsers.tasks.parse_manufactures",
+        "schedule": 86400.0,  # Every 24 hours
+    },
+    # Сканирование папки FNS - каждые 5 минут
+    "scan-fns-directory": {
+        "task": "apps.parsers.tasks.scan_fns_directory",
+        "schedule": 300.0,  # Every 5 minutes
    },
 }

-app.conf.timezone = "UTC"
+app.conf.timezone = "Europe/Moscow"


@app.task(bind=True)
--- a/src/config/settings/base.py
+++ b/src/config/settings/base.py
@@ -4,6 +4,7 @@ Base settings for Django project.
 Generated by 'django-admin startproject' using Django 3.2.25.
 """

+from datetime import timedelta
 from pathlib import Path

 from decouple import Config, RepositoryEnv
@@ -45,6 +46,7 @@ if isinstance(ALLOWED_HOSTS, str):

 # Application definition
 INSTALLED_APPS = [
+    "jazzmin",  # Django Jazzmin - modern admin theme (must be before admin)
    "django.contrib.admin",
    "django.contrib.auth",
    "django.contrib.contenttypes",
@@ -61,12 +63,113 @@ INSTALLED_APPS = [
    # Local apps
    "apps.core",
    "apps.user",
+    "apps.organization",
+    "apps.form_1",
+    "apps.form_2",
+    "apps.form_3",
+    "apps.form_4",
+    "apps.form_5",
+    "apps.form_6",
 ]

+# Jazzmin Admin Configuration
+JAZZMIN_SETTINGS = {
+    # Title
+    "site_title": "State Corp Admin",
+    "site_header": "State Corp",
+    "site_brand": "State Corp",
+    "site_logo": None,
+    "login_logo": None,
+    "login_logo_dark": None,
+    "site_logo_classes": "img-circle",
+    "site_icon": None,
+    "welcome_sign": "Добро пожаловать в панель управления",
+    "copyright": "State Corp Backend",
+    # Search
+    "search_model": ["user.User", "parsers.IndustrialCertificateRecord"],
+    # User menu
+    "topmenu_links": [
+        {"name": "Главная", "url": "admin:index", "permissions": ["auth.view_user"]},
+        {"name": "API Docs", "url": "/api/docs/", "new_window": True},
+        {"model": "user.User"},
+    ],
+    # Side menu
+    "show_sidebar": True,
+    "navigation_expanded": True,
+    "hide_apps": ["django_celery_results"],
+    "hide_models": [],
+    "order_with_respect_to": [
+        "user",
+        "parsers",
+        "core",
+        "django_celery_beat",
+    ],
+    # Icons (Font Awesome)
+    "icons": {
+        "auth": "fas fa-users-cog",
+        "auth.Group": "fas fa-users",
+        "user.User": "fas fa-user",
+        "user.Profile": "fas fa-id-card",
+        "core.BackgroundJob": "fas fa-tasks",
+        "django_celery_beat.PeriodicTask": "fas fa-clock",
+        "django_celery_beat.CrontabSchedule": "fas fa-calendar-alt",
+        "django_celery_beat.IntervalSchedule": "fas fa-stopwatch",
+        "django_celery_results.TaskResult": "fas fa-clipboard-check",
+    },
+    "default_icon_parents": "fas fa-chevron-circle-right",
+    "default_icon_children": "fas fa-circle",
+    # Related modal
+    "related_modal_active": True,
+    # UI Tweaks
+    "custom_css": None,
+    "custom_js": None,
+    "use_google_fonts_cdn": True,
+    "show_ui_builder": False,
+    # Change view
+    "changeform_format": "horizontal_tabs",
+    "changeform_format_overrides": {
+        "user.User": "collapsible",
+        "parsers.IndustrialCertificateRecord": "vertical_tabs",
+    },
+}
+
+JAZZMIN_UI_TWEAKS = {
+    "navbar_small_text": False,
+    "footer_small_text": False,
+    "body_small_text": False,
+    "brand_small_text": False,
+    "brand_colour": "navbar-primary",
+    "accent": "accent-primary",
+    "navbar": "navbar-dark",
+    "no_navbar_border": False,
+    "navbar_fixed": True,
+    "layout_boxed": False,
+    "footer_fixed": False,
+    "sidebar_fixed": True,
+    "sidebar": "sidebar-dark-primary",
+    "sidebar_nav_small_text": False,
+    "sidebar_disable_expand": False,
+    "sidebar_nav_child_indent": False,
+    "sidebar_nav_compact_style": False,
+    "sidebar_nav_legacy_style": False,
+    "sidebar_nav_flat_style": False,
+    "theme": "default",
+    "dark_mode_theme": "darkly",
+    "button_classes": {
+        "primary": "btn-primary",
+        "secondary": "btn-secondary",
+        "info": "btn-info",
+        "warning": "btn-warning",
+        "danger": "btn-danger",
+        "success": "btn-success",
+    },
+}
+
 MIDDLEWARE = [
    "apps.core.middleware.RequestIDMiddleware",
    "corsheaders.middleware.CorsMiddleware",
    "django.middleware.security.SecurityMiddleware",
+    "whitenoise.middleware.WhiteNoiseMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.middleware.csrf.CsrfViewMiddleware",
@@ -104,9 +207,6 @@ DATABASES = {
        "PASSWORD": get_env("POSTGRES_PASSWORD", "project_password"),
        "HOST": get_env("POSTGRES_HOST", "db"),
        "PORT": int(get_env("POSTGRES_PORT", "5432")),
-        "OPTIONS": {
-            "charset": "utf8mb4",
-        },
    },
 }

@@ -122,6 +222,24 @@ CACHES = {
 }


+# =============================================================================
+# PARSERS SETTINGS
+# =============================================================================
+
+# Zakupki.gov.ru API Token (получить через Госуслуги)
+ZAKUPKI_TOKEN = get_env("ZAKUPKI_TOKEN", "")
+
+# FNS file lock TTL (seconds)
+FNS_LOCK_TTL_SECONDS = int(get_env("FNS_LOCK_TTL_SECONDS", "3600"))
+
+# Proxy list for parsers (comma-separated)
+PARSER_PROXIES = get_env("PARSER_PROXIES", "")
+if isinstance(PARSER_PROXIES, str) and PARSER_PROXIES:
+    PARSER_PROXIES = [p.strip() for p in PARSER_PROXIES.split(",") if p.strip()]
+else:
+    PARSER_PROXIES = []
+
+
 # Password validation
 AUTH_PASSWORD_VALIDATORS = [
    {
@@ -160,6 +278,9 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 # Custom user model
 AUTH_USER_MODEL = "user.User"

+# Login URL for drf-yasg and DRF browsable API
+LOGIN_URL = "/auth/login/"
+
 # REST Framework settings
 REST_FRAMEWORK = {
    "DEFAULT_AUTHENTICATION_CLASSES": [
@@ -193,7 +314,6 @@ REST_FRAMEWORK = {
 }

 # JWT settings
-from datetime import timedelta

 SIMPLE_JWT = {
    "ACCESS_TOKEN_LIFETIME": timedelta(minutes=60),
@@ -230,6 +350,24 @@ if isinstance(CORS_ALLOWED_ORIGINS, str):
    CORS_ALLOWED_ORIGINS = CORS_ALLOWED_ORIGINS.split(",")
 CORS_ALLOW_CREDENTIALS = True

+# =============================================================================
+# SWAGGER SETTINGS (drf-yasg)
+# =============================================================================
+SWAGGER_SETTINGS = {
+    "SECURITY_DEFINITIONS": {
+        "Bearer": {
+            "type": "apiKey",
+            "name": "Authorization",
+            "in": "header",
+            "description": "JWT авторизация. Формат: Bearer <access_token>",
+        }
+    },
+    "USE_SESSION_AUTH": True,
+    "PERSIST_AUTH": True,
+    "REFETCH_SCHEMA_WITH_AUTH": True,
+    "REFETCH_SCHEMA_ON_LOGOUT": True,
+}
+
 # Logging configuration
 LOGGING = {
    "version": 1,
@@ -269,3 +407,23 @@ LOGGING = {
        },
    },
 }
+
+# =============================================================================
+# FNS Parser Settings
+# =============================================================================
+
+# Directory for watching incoming FNS files
+FNS_WATCH_DIRECTORY = BASE_DIR / "input" / "fns"
+
+# Directory for processed files (moved after successful processing)
+FNS_PROCESSED_DIRECTORY = BASE_DIR / "input" / "fns" / "processed"
+
+# Directory for failed files (moved after failed processing)
+FNS_FAILED_DIRECTORY = BASE_DIR / "input" / "fns" / "failed"
+
+# =============================================================================
+# Checko API Settings (checko.ru)
+# =============================================================================
+
+# API key for Checko.ru service
+CHECKO_API_KEY = get_env("CHECKO_API_KEY", "")
--- a/src/config/settings/development.py
+++ b/src/config/settings/development.py
@@ -1,44 +1,49 @@
+import os
+
 from .base import *

 # Development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/

 # SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "django-insecure-development-key-change-in-production"
+SECRET_KEY = os.getenv(
+    "SECRET_KEY", "django-insecure-development-key-change-in-production"
+)

 # SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
+DEBUG = os.getenv("DEBUG", "True").lower() in ("true", "1", "yes")

-ALLOWED_HOSTS = ["localhost", "127.0.0.1", "0.0.0.0", "testserver"]
+ALLOWED_HOSTS = ["localhost", "127.0.0.1", "0.0.0.0", "testserver", "*"]  # noqa: S104

 # Database for development
 DATABASES = {
    "default": {
        "ENGINE": "django.db.backends.postgresql",
-        "NAME": "project_dev",
-        "USER": "postgres",
-        "PASSWORD": "postgres",
-        "HOST": "localhost",
-        "PORT": "5432",
+        "NAME": os.getenv("POSTGRES_DB", "project_dev"),
+        "USER": os.getenv("POSTGRES_USER", "postgres"),
+        "PASSWORD": os.getenv("POSTGRES_PASSWORD", "postgres"),
+        "HOST": os.getenv("POSTGRES_HOST", "localhost"),
+        "PORT": os.getenv("POSTGRES_PORT", "5432"),
    }
 }

 # Celery Configuration for Development
-CELERY_BROKER_URL = "redis://localhost:6379/0"
-CELERY_RESULT_BACKEND = "redis://localhost:6379/0"
+CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+CELERY_RESULT_BACKEND = os.getenv("CELERY_RESULT_BACKEND", "redis://localhost:6379/0")
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
 CELERY_RESULT_SERIALIZER = "json"
-CELERY_TIMEZONE = "UTC"
+CELERY_TIMEZONE = "Europe/Moscow"

 # Email backend for development
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

 # Cache configuration for development
+REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/1")
 CACHES = {
    "default": {
        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": "redis://127.0.0.1:6379/1",
+        "LOCATION": REDIS_URL,
        "OPTIONS": {
            "CLIENT_CLASS": "django_redis.client.DefaultClient",
        },
--- a/src/config/urls.py
+++ b/src/config/urls.py
@@ -15,11 +15,24 @@ from rest_framework import permissions
 # Swagger schema view
 schema_view = get_schema_view(
    openapi.Info(
-        title="Mostovik API",
+        title="State Corp API",
        default_version="v1",
-        description="API documentation for Mostovik project",
-        terms_of_service="https://www.google.com/policies/terms/",
-        contact=openapi.Contact(email="contact@mostovik.local"),
+        description="""
+## API документация для проекта State Corp
+
+### Авторизация
+Для доступа к защищённым эндпоинтам используйте JWT токен:
+1. Получите токен через `POST /api/v1/users/login/`
+2. Добавьте заголовок: `Authorization: Bearer <access_token>`
+
+### Обновление токена
+Используйте `POST /api/v1/users/token/refresh/` с refresh токеном.
+
+### Парсеры
+API предоставляет только чтение данных (GET, GET list).
+Добавление и удаление записей происходит через парсеры и админку.
+        """,
+        contact=openapi.Contact(email="contact@state-corp.local"),
        license=openapi.License(name="BSD License"),
    ),
    public=True,