feat(parsers): add proverki.gov.ru parser with sync_inspections task
Some checks failed
CI/CD Pipeline / Build Docker Images (push) Blocked by required conditions
CI/CD Pipeline / Push to Gitea Registry (push) Blocked by required conditions
CI/CD Pipeline / Code Quality Checks (push) Failing after 3m55s
CI/CD Pipeline / Run Tests (push) Failing after 3h11m38s

- Add InspectionRecord model with is_federal_law_248, data_year, data_month fields
- Add ProverkiClient with Playwright support for JS-rendered portal
- Add streaming XML parser for large files (>50MB)
- Add sync_inspections task with incremental loading logic
  - Starts from 01.01.2025 if DB is empty
  - Loads both FZ-294 and FZ-248 inspections
  - Stops after 2 consecutive empty months
- Add InspectionService methods: get_last_loaded_period, has_data_for_period
- Add Minpromtorg parsers (certificates, manufacturers)
- Add Django Admin for parser models
- Update README with parsers documentation and changelog
This commit is contained in:
2026-01-21 20:16:25 +01:00
parent f121445313
commit 199d871923
45 changed files with 6810 additions and 97 deletions

View File

@@ -1,21 +1,19 @@
version: '3.8'
services:
db:
image: postgres:15.10
container_name: project_db
container_name: mostovik_db
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-project_dev}
POSTGRES_DB: ${POSTGRES_DB:-mostovik_dev}
POSTGRES_USER: ${POSTGRES_USER:-postgres}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
volumes:
- postgres_data:/var/lib/postgresql/data
- ./data/db:/var/lib/postgresql/data
- ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
ports:
- "5432:5432"
networks:
- project_network
- mostovik_network
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 30s
@@ -24,14 +22,14 @@ services:
redis:
image: redis:7-alpine
container_name: project_redis
container_name: mostovik_redis
restart: unless-stopped
ports:
- "6379:6379"
volumes:
- redis_data:/data
- ./data/redis:/data
networks:
- project_network
- mostovik_network
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 30s
@@ -42,7 +40,7 @@ services:
build:
context: .
dockerfile: docker/Dockerfile.web
container_name: project_web
container_name: mostovik_web
restart: unless-stopped
depends_on:
db:
@@ -54,11 +52,12 @@ services:
- SECRET_KEY=${SECRET_KEY:-django-insecure-development-key}
- POSTGRES_HOST=db
- POSTGRES_PORT=5432
- POSTGRES_DB=${POSTGRES_DB:-project_dev}
- POSTGRES_DB=${POSTGRES_DB:-mostovik_dev}
- POSTGRES_USER=${POSTGRES_USER:-postgres}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
- REDIS_URL=redis://redis:6379/0
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0
volumes:
- ./src:/app/src
- ./logs:/app/logs
@@ -67,7 +66,7 @@ services:
ports:
- "8000:8000"
networks:
- project_network
- mostovik_network
command: >
sh -c "python src/manage.py migrate &&
python src/manage.py collectstatic --noinput &&
@@ -77,7 +76,7 @@ services:
build:
context: .
dockerfile: docker/Dockerfile.celery
container_name: project_celery_worker
container_name: mostovik_celery_worker
restart: unless-stopped
depends_on:
db:
@@ -88,23 +87,24 @@ services:
- DEBUG=${DEBUG:-True}
- POSTGRES_HOST=db
- POSTGRES_PORT=5432
- POSTGRES_DB=${POSTGRES_DB:-project_dev}
- POSTGRES_DB=${POSTGRES_DB:-mostovik_dev}
- POSTGRES_USER=${POSTGRES_USER:-postgres}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
- REDIS_URL=redis://redis:6379/0
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0
volumes:
- ./src:/app/src
- ./logs:/app/logs
networks:
- project_network
- mostovik_network
command: celery -A config worker --loglevel=info
celery_beat:
build:
context: .
dockerfile: docker/Dockerfile.celery
container_name: project_celery_beat
container_name: mostovik_celery_beat
restart: unless-stopped
depends_on:
db:
@@ -115,22 +115,19 @@ services:
- DEBUG=${DEBUG:-True}
- POSTGRES_HOST=db
- POSTGRES_PORT=5432
- POSTGRES_DB=${POSTGRES_DB:-project_dev}
- POSTGRES_DB=${POSTGRES_DB:-mostovik_dev}
- POSTGRES_USER=${POSTGRES_USER:-postgres}
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
- REDIS_URL=redis://redis:6379/0
- CELERY_BROKER_URL=redis://redis:6379/0
- CELERY_RESULT_BACKEND=redis://redis:6379/0
volumes:
- ./src:/app/src
- ./logs:/app/logs
networks:
- project_network
- mostovik_network
command: celery -A config beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler
volumes:
postgres_data:
redis_data:
networks:
project_network:
driver: bridge
mostovik_network:
driver: bridge