avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(parsers): add proverki.gov.ru parser with sync_inspections task
Some checks failed
CI/CD Pipeline / Build Docker Images (push) Blocked by required conditions
Details
CI/CD Pipeline / Push to Gitea Registry (push) Blocked by required conditions
Details
CI/CD Pipeline / Code Quality Checks (push) Failing after 3m55s
Details
CI/CD Pipeline / Run Tests (push) Failing after 3h11m38s
Details

Browse Source 
- Add InspectionRecord model with is_federal_law_248, data_year, data_month fields
- Add ProverkiClient with Playwright support for JS-rendered portal
- Add streaming XML parser for large files (>50MB)
- Add sync_inspections task with incremental loading logic
  - Starts from 01.01.2025 if DB is empty
  - Loads both FZ-294 and FZ-248 inspections
  - Stops after 2 consecutive empty months
- Add InspectionService methods: get_last_loaded_period, has_data_for_period
- Add Minpromtorg parsers (certificates, manufacturers)
- Add Django Admin for parser models
- Update README with parsers documentation and changelog
This commit is contained in:
Aleksandr Meshchriakov
2026-01-21 20:16:25 +01:00
parent f121445313
commit 199d871923
45 changed files with 6810 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

262
requirements.txt
View File

@@ -1,11 +1,15 @@
# This file was autogenerated by uv via the following command:
# uv pip compile pyproject.toml --output-file=requirements.txt
# uv export --no-hashes
alabaster==0.7.16
# via sphinx
amqp==5.3.1
# via kombu
asgiref==3.11.0
# via
# django
# django-cors-headers
async-timeout==5.0.1 ; python_full_version < '3.11.3'
# via redis
attrs==25.4.0
# via
# outcome
@@ -14,49 +18,69 @@ attrs==25.4.0
# twisted
automat==25.4.16
# via twisted
babel==2.17.0
# via sphinx
bandit==1.7.5
beautifulsoup4==4.12.3
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
billiard==4.2.4
# via celery
black==23.12.1
celery==5.3.6
# via
# mostovik-backend (pyproject.toml)
# django-celery-beat
# django-celery-results
# flower
# mostovik-backend
certifi==2026.1.4
# via
# requests
# selenium
cffi==2.0.0
# via cryptography
cffi==2.0.0 ; (implementation_name != 'pypy' and os_name == 'nt') or platform_python_implementation != 'PyPy'
# via
# cryptography
# gevent
# trio
cfgv==3.5.0
# via pre-commit
charset-normalizer==3.4.4
# via requests
click==8.1.7
# via
# black
# celery
# click-didyoumean
# click-plugins
# click-repl
# typer
click-didyoumean==0.3.1
# via celery
click-plugins==1.1.1.2
# via celery
click-repl==0.3.0
# via celery
colorama==0.4.6 ; sys_platform == 'win32'
# via
# bandit
# click
# pytest
# sphinx
constantly==23.10.4
# via twisted
coreapi==2.3.3
# via
# mostovik-backend (pyproject.toml)
# django-rest-swagger
# mostovik-backend
# openapi-codec
coreschema==0.0.4
# via coreapi
coverage==7.4.0
# via pytest-cov
cron-descriptor==2.0.6
# via django-celery-beat
cryptography==42.0.5
# via
# mostovik-backend (pyproject.toml)
# mostovik-backend
# pyopenssl
# scrapy
# service-identity
@@ -66,59 +90,108 @@ cssselect==1.3.0
# scrapy
defusedxml==0.7.1
# via scrapy
distlib==0.4.0
# via virtualenv
django==3.2.25
# via
# mostovik-backend (pyproject.toml)
# django-celery-beat
# django-celery-results
# django-cors-headers
# django-debug-toolbar
# django-extensions
# django-filter
# django-jazzmin
# django-redis
# django-stubs
# django-stubs-ext
# django-timezone-field
# djangorestframework
# djangorestframework-simplejwt
# drf-yasg
# model-bakery
# mostovik-backend
django-celery-beat==2.6.0
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
django-celery-results==2.5.1
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
django-cors-headers==4.3.1
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
django-debug-toolbar==4.2.0
django-extensions==3.2.3
django-filter==23.5
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
django-jazzmin==2.6.2
# via mostovik-backend
django-redis==5.4.0
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
django-rest-swagger==2.2.0
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
django-stubs==4.2.7
django-stubs-ext==5.2.9
# via django-stubs
django-timezone-field==7.2.1
# via django-celery-beat
djangorestframework==3.14.0
# via
# mostovik-backend (pyproject.toml)
# django-rest-swagger
# djangorestframework-simplejwt
# drf-yasg
# mostovik-backend
djangorestframework-simplejwt==5.3.1
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
docutils==0.20.1
# via
# sphinx
# sphinx-rtd-theme
drf-yasg==1.21.10
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
et-xmlfile==2.0.0
# via openpyxl
factory-boy==3.3.0
# via mostovik-backend
faker==40.1.2
# via
# factory-boy
# mostovik-backend
filelock==3.20.3
# via tldextract
# via
# tldextract
# virtualenv
flake8==6.1.0
flower==2.0.1
gevent==23.9.1
gitdb==4.0.12
# via gitpython
gitpython==3.1.46
# via bandit
greenlet==3.3.0
# via
# gevent
# playwright
gunicorn==21.2.0
h11==0.16.0
# via wsproto
humanize==4.15.0
# via flower
hyperlink==21.0.0
# via twisted
identify==2.6.16
# via pre-commit
idna==3.11
# via
# hyperlink
# requests
# tldextract
# trio
imagesize==1.4.1
# via sphinx
incremental==24.11.0
# via twisted
inflection==0.5.1
# via drf-yasg
iniconfig==2.3.0
# via pytest
isort==5.13.2
itemadapter==0.13.1
# via
# itemloaders
@@ -128,7 +201,9 @@ itemloaders==1.3.2
itypes==1.2.0
# via coreapi
jinja2==3.1.6
# via coreschema
# via
# coreschema
# sphinx
jmespath==1.0.1
# via
# itemloaders
@@ -139,97 +214,157 @@ lxml==6.0.2
# via
# parsel
# scrapy
markdown-it-py==3.0.0
# via rich
markupsafe==3.0.3
# via jinja2
# via
# jinja2
# werkzeug
mccabe==0.7.0
# via flake8
mdurl==0.1.2
# via markdown-it-py
model-bakery==1.17.0
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
mypy==1.8.0
mypy-extensions==1.1.0
# via
# black
# mypy
nodeenv==1.10.0
# via pre-commit
numpy==1.24.4
# via
# mostovik-backend (pyproject.toml)
# mostovik-backend
# pandas
openapi-codec==1.3.2
# via django-rest-swagger
openpyxl==3.1.5
# via mostovik-backend
outcome==1.3.0.post0
# via
# trio
# trio-websocket
packaging==25.0
# via
# black
# drf-yasg
# gunicorn
# incremental
# kombu
# parsel
# pytest
# scrapy
# sphinx
pandas==2.0.3
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
parsel==1.10.0
# via
# itemloaders
# scrapy
pathspec==1.0.3
# via black
pillow==12.1.0
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
platformdirs==4.5.1
# via
# black
# virtualenv
playwright==1.57.0
# via mostovik-backend
pluggy==1.6.0
# via pytest
pre-commit==3.6.0
prometheus-client==0.24.1
# via flower
prompt-toolkit==3.0.52
# via click-repl
protego==0.5.0
# via scrapy
psycopg2-binary==2.9.9
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
pyasn1==0.6.2
# via
# pyasn1-modules
# service-identity
pyasn1-modules==0.4.2
# via service-identity
pycparser==2.23
pycodestyle==2.11.1
# via flake8
pycparser==2.23 ; (implementation_name != 'PyPy' and implementation_name != 'pypy' and os_name == 'nt') or (implementation_name != 'PyPy' and platform_python_implementation != 'PyPy')
# via cffi
pydispatcher==2.0.7
pydispatcher==2.0.7 ; platform_python_implementation == 'CPython'
# via scrapy
pyee==13.0.0
# via playwright
pyflakes==3.1.0
# via flake8
pygments==2.19.2
# via
# rich
# sphinx
pyjwt==2.10.1
# via djangorestframework-simplejwt
pyopenssl==25.1.0
# via scrapy
pypydispatcher==2.1.2 ; platform_python_implementation == 'PyPy'
# via scrapy
pysocks==1.7.1
# via urllib3
pytest==7.4.4
# via
# pytest-cov
# pytest-django
pytest-cov==4.1.0
pytest-django==4.7.0
python-crontab==3.3.0
# via django-celery-beat
python-dateutil==2.8.2
# via
# mostovik-backend (pyproject.toml)
# celery
# mostovik-backend
# pandas
python-decouple==3.8
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
python-dotenv==1.0.1
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
python-json-logger==2.0.7
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
pytz==2024.1
# via
# mostovik-backend (pyproject.toml)
# django
# djangorestframework
# drf-yasg
# flower
# mostovik-backend
# pandas
pyyaml==6.0.3
# via drf-yasg
# via
# bandit
# drf-yasg
# pre-commit
queuelib==1.8.0
# via scrapy
redis==5.0.3
# via
# mostovik-backend (pyproject.toml)
# django-redis
# mostovik-backend
requests==2.31.0
# via
# mostovik-backend (pyproject.toml)
# coreapi
# mostovik-backend
# requests-file
# sphinx
# tldextract
requests-file==3.0.1
# via tldextract
rich==14.2.0
# via bandit
ruff==0.1.14
scrapy==2.11.2
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
selenium==4.17.2
# via mostovik-backend (pyproject.toml)
# via mostovik-backend
service-identity==24.2.0
# via scrapy
setuptools==80.9.0
@@ -238,16 +373,47 @@ simplejson==3.20.2
# via django-rest-swagger
six==1.17.0
# via python-dateutil
smmap==5.0.2
# via gitdb
sniffio==1.3.1
# via trio
snowballstemmer==3.0.1
# via sphinx
sortedcontainers==2.4.0
# via trio
soupsieve==2.8.2
# via beautifulsoup4
sphinx==7.2.6
# via
# sphinx-rtd-theme
# sphinxcontrib-jquery
sphinx-rtd-theme==2.0.0
sphinxcontrib-applehelp==2.0.0
# via sphinx
sphinxcontrib-devhelp==2.0.0
# via sphinx
sphinxcontrib-htmlhelp==2.1.0
# via sphinx
sphinxcontrib-jquery==4.1
# via sphinx-rtd-theme
sphinxcontrib-jsmath==1.0.1
# via sphinx
sphinxcontrib-qthelp==2.0.0
# via sphinx
sphinxcontrib-serializinghtml==2.0.0
# via sphinx
sqlparse==0.5.5
# via django
# via
# django
# django-debug-toolbar
stevedore==5.6.0
# via bandit
tldextract==5.3.1
# via scrapy
tomli==2.4.0 ; python_full_version <= '3.11'
# via coverage
tornado==6.5.4
# via flower
trio==0.32.0
# via
# selenium
@@ -256,16 +422,28 @@ trio-websocket==0.12.2
# via selenium
twisted==25.5.0
# via scrapy
typer==0.9.0
types-pytz==2025.2.0.20251108
# via django-stubs
types-pyyaml==6.0.12.20250915
# via django-stubs
types-requests==2.31.0.20240125
typing-extensions==4.15.0
# via
# cron-descriptor
# django-stubs
# django-stubs-ext
# mypy
# pyee
# pyopenssl
# selenium
# twisted
# typer
tzdata==2025.3
# via
# celery
# django-celery-beat
# faker
# kombu
# pandas
uritemplate==4.2.0
@@ -276,20 +454,28 @@ urllib3==2.6.3
# via
# requests
# selenium
# types-requests
vine==5.1.0
# via
# amqp
# celery
# kombu
virtualenv==20.36.1
# via pre-commit
w3lib==2.3.1
# via
# parsel
# scrapy
watchdog==3.0.0
wcwidth==0.2.14
# via prompt-toolkit
werkzeug==3.0.1
wsproto==1.3.2
# via trio-websocket
zope-event==6.1
# via gevent
zope-interface==8.2
# via
# gevent
# scrapy
# twisted