feat(parsers): add proverki.gov.ru parser with sync_inspections task
Some checks failed
CI/CD Pipeline / Build Docker Images (push) Blocked by required conditions
CI/CD Pipeline / Push to Gitea Registry (push) Blocked by required conditions
CI/CD Pipeline / Code Quality Checks (push) Failing after 3m55s
CI/CD Pipeline / Run Tests (push) Failing after 3h11m38s
Some checks failed
CI/CD Pipeline / Build Docker Images (push) Blocked by required conditions
CI/CD Pipeline / Push to Gitea Registry (push) Blocked by required conditions
CI/CD Pipeline / Code Quality Checks (push) Failing after 3m55s
CI/CD Pipeline / Run Tests (push) Failing after 3h11m38s
- Add InspectionRecord model with is_federal_law_248, data_year, data_month fields - Add ProverkiClient with Playwright support for JS-rendered portal - Add streaming XML parser for large files (>50MB) - Add sync_inspections task with incremental loading logic - Starts from 01.01.2025 if DB is empty - Loads both FZ-294 and FZ-248 inspections - Stops after 2 consecutive empty months - Add InspectionService methods: get_last_loaded_period, has_data_for_period - Add Minpromtorg parsers (certificates, manufacturers) - Add Django Admin for parser models - Update README with parsers documentation and changelog
This commit is contained in:
@@ -1,21 +1,19 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:15.10
|
||||
container_name: project_db
|
||||
container_name: mostovik_db
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-project_dev}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-mostovik_dev}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-postgres}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ./data/db:/var/lib/postgresql/data
|
||||
- ./docker/postgres/init.sql:/docker-entrypoint-initdb.d/init.sql
|
||||
ports:
|
||||
- "5432:5432"
|
||||
networks:
|
||||
- project_network
|
||||
- mostovik_network
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 30s
|
||||
@@ -24,14 +22,14 @@ services:
|
||||
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
container_name: project_redis
|
||||
container_name: mostovik_redis
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "6379:6379"
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
- ./data/redis:/data
|
||||
networks:
|
||||
- project_network
|
||||
- mostovik_network
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 30s
|
||||
@@ -42,7 +40,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile.web
|
||||
container_name: project_web
|
||||
container_name: mostovik_web
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
db:
|
||||
@@ -54,11 +52,12 @@ services:
|
||||
- SECRET_KEY=${SECRET_KEY:-django-insecure-development-key}
|
||||
- POSTGRES_HOST=db
|
||||
- POSTGRES_PORT=5432
|
||||
- POSTGRES_DB=${POSTGRES_DB:-project_dev}
|
||||
- POSTGRES_DB=${POSTGRES_DB:-mostovik_dev}
|
||||
- POSTGRES_USER=${POSTGRES_USER:-postgres}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
|
||||
- REDIS_URL=redis://redis:6379/0
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
volumes:
|
||||
- ./src:/app/src
|
||||
- ./logs:/app/logs
|
||||
@@ -67,7 +66,7 @@ services:
|
||||
ports:
|
||||
- "8000:8000"
|
||||
networks:
|
||||
- project_network
|
||||
- mostovik_network
|
||||
command: >
|
||||
sh -c "python src/manage.py migrate &&
|
||||
python src/manage.py collectstatic --noinput &&
|
||||
@@ -77,7 +76,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile.celery
|
||||
container_name: project_celery_worker
|
||||
container_name: mostovik_celery_worker
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
db:
|
||||
@@ -88,23 +87,24 @@ services:
|
||||
- DEBUG=${DEBUG:-True}
|
||||
- POSTGRES_HOST=db
|
||||
- POSTGRES_PORT=5432
|
||||
- POSTGRES_DB=${POSTGRES_DB:-project_dev}
|
||||
- POSTGRES_DB=${POSTGRES_DB:-mostovik_dev}
|
||||
- POSTGRES_USER=${POSTGRES_USER:-postgres}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
|
||||
- REDIS_URL=redis://redis:6379/0
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
volumes:
|
||||
- ./src:/app/src
|
||||
- ./logs:/app/logs
|
||||
networks:
|
||||
- project_network
|
||||
- mostovik_network
|
||||
command: celery -A config worker --loglevel=info
|
||||
|
||||
celery_beat:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: docker/Dockerfile.celery
|
||||
container_name: project_celery_beat
|
||||
container_name: mostovik_celery_beat
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
db:
|
||||
@@ -115,22 +115,19 @@ services:
|
||||
- DEBUG=${DEBUG:-True}
|
||||
- POSTGRES_HOST=db
|
||||
- POSTGRES_PORT=5432
|
||||
- POSTGRES_DB=${POSTGRES_DB:-project_dev}
|
||||
- POSTGRES_DB=${POSTGRES_DB:-mostovik_dev}
|
||||
- POSTGRES_USER=${POSTGRES_USER:-postgres}
|
||||
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-postgres}
|
||||
- REDIS_URL=redis://redis:6379/0
|
||||
- CELERY_BROKER_URL=redis://redis:6379/0
|
||||
- CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
volumes:
|
||||
- ./src:/app/src
|
||||
- ./logs:/app/logs
|
||||
networks:
|
||||
- project_network
|
||||
- mostovik_network
|
||||
command: celery -A config beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
networks:
|
||||
project_network:
|
||||
driver: bridge
|
||||
mostovik_network:
|
||||
driver: bridge
|
||||
|
||||
Reference in New Issue
Block a user