avm/mostovik-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Collect sources for State Corp corporations #38

Merged
avm merged 6 commits from feature/state-corp-source-scope into main 2026-07-24 11:06:21 +03:00
Conversation 0 Commits 6 Files Changed 7 +237 -14
7 changed files with 237 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/apps/parsers/serializers.py
View File

@@ -701,6 +701,8 @@ class ParserResultQuerySerializer(serializers.Serializer):
batch_id = serializers.IntegerField(required=False, min_value=1)
status = serializers.CharField(required=False, allow_blank=True)
record_date = serializers.CharField(required=False, allow_blank=True)
date_from = serializers.DateField(required=False)
date_to = serializers.DateField(required=False)
search = serializers.CharField(required=False, allow_blank=True)
ordering = serializers.CharField(required=False, allow_blank=True)
include_payload = serializers.BooleanField(required=False, default=True)
@@ -712,6 +714,14 @@ class ParserResultQuerySerializer(serializers.Serializer):
attrs["page_size"] = attrs["limit"]
if attrs.get("batch_id") and not attrs.get("load_batch"):
attrs["load_batch"] = attrs["batch_id"]
if (
attrs.get("date_from")
and attrs.get("date_to")
and attrs["date_from"] > attrs["date_to"]
):
raise serializers.ValidationError(
{"date_to": "date_to must be greater than or equal to date_from"}
)
return attrs

16
src/apps/parsers/tasks.py
View File

@@ -178,6 +178,17 @@ def _resolve_lookup_limit(
return max(resolved, 0)
def _rosatom_roscosmos_scope_query() -> Q:
"""Вернуть условие принадлежности к контуру Росатома или Роскосмоса."""
return (
Q(gk_name__icontains="Росатом")
| Q(gk_name__icontains="Роскосмос")
| Q(gk_code__iexact="rosatom")
| Q(gk_code__iexact="roscosmos")
| Q(gk_code__iexact="roskosmos")
)
def _active_registry_lookup_targets(
*,
limit: int | None = None,
@@ -186,7 +197,9 @@ def _active_registry_lookup_targets(
organization_ids: list[str] | None = None,
) -> list[RegistryLookupTarget]:
"""Вернуть организации, которые сейчас состоят хотя бы в одном реестре."""
queryset = SourceOrganization.objects.filter(opk_registry_membership=True)
queryset = SourceOrganization.objects.filter(
Q(opk_registry_membership=True) | _rosatom_roscosmos_scope_query()
)
if organization_ids is not None:
queryset = queryset.filter(uid__in=organization_ids)
if require_inn:
@@ -3977,6 +3990,7 @@ def _active_fns_registry_inns(*, limit: int) -> list[str]:
Q(opk_registry_membership=True)
| Q(goz_participation=True)
| ~Q(ropk_num="")
| _rosatom_roscosmos_scope_query()
)
.exclude(inn="")
.order_by("inn")

84
src/apps/parsers/views.py
View File

@@ -10,6 +10,7 @@ import json
import uuid
from collections import defaultdict
from apps.core.filters import BaseFilterSet
from apps.core.openapi import CommonResponses, ErrorResponses, swagger_tag
from apps.core.response import api_error_response, api_response
from apps.core.serializers import BackgroundJobListSerializer
@@ -73,6 +74,7 @@ from django.db.models.functions import Cast, Lower
from django.http import HttpResponse
from django.utils.text import get_valid_filename
from django_celery_beat.models import CrontabSchedule, IntervalSchedule, PeriodicTask
from django_filters import rest_framework as filters
from drf_yasg import openapi
from drf_yasg.inspectors import SwaggerAutoSchema
from drf_yasg.utils import no_body, swagger_auto_schema
@@ -331,6 +333,20 @@ ORDERING_PARAM = openapi.Parameter(
description="Сортировка",
type=openapi.TYPE_STRING,
)
DATE_FROM_PARAM = openapi.Parameter(
"date_from",
openapi.IN_QUERY,
description="Дата записи с которой включительно отбирать результаты (YYYY-MM-DD)",
type=openapi.TYPE_STRING,
format=openapi.FORMAT_DATE,
)
DATE_TO_PARAM = openapi.Parameter(
"date_to",
openapi.IN_QUERY,
description="Дата записи по которую включительно отбирать результаты (YYYY-MM-DD)",
type=openapi.TYPE_STRING,
format=openapi.FORMAT_DATE,
)
INCLUDE_PAYLOAD_PARAM = openapi.Parameter(
"include_payload",
openapi.IN_QUERY,
@@ -355,6 +371,8 @@ RESULT_LIST_PARAMS = [
OGRN_PARAM,
LOAD_BATCH_PARAM,
STATUS_PARAM,
DATE_FROM_PARAM,
DATE_TO_PARAM,
SEARCH_PARAM,
ORDERING_PARAM,
INCLUDE_PAYLOAD_PARAM,
@@ -790,6 +808,32 @@ class IndustrialProductViewSet(ReadOnlyModelViewSet):
# =============================================================================
class InspectionFilter(BaseFilterSet):
"""Filters inspections by their normalized start date."""
date_from = filters.DateFilter(
field_name="start_date_normalized",
lookup_expr="gte",
)
date_to = filters.DateFilter(
field_name="start_date_normalized",
lookup_expr="lte",
)
class Meta:
model = InspectionRecord
fields = [
"inn",
"ogrn",
"registration_number",
"is_federal_law_248",
"data_year",
"data_month",
"load_batch",
"registry_organization",
]
class InspectionViewSet(ReadOnlyModelViewSet):
"""
API для просмотра проверок из Единого реестра проверок.
@@ -804,16 +848,15 @@ class InspectionViewSet(ReadOnlyModelViewSet):
).order_by("-created_at")
serializer_class = InspectionSerializer
permission_classes = [IsAuthenticated]
filterset_fields = [
"inn",
"ogrn",
"registration_number",
"is_federal_law_248",
"data_year",
"data_month",
"load_batch",
"registry_organization",
filterset_class = InspectionFilter
ordering_fields = [
"id",
"start_date_normalized",
"end_date_normalized",
"created_at",
"updated_at",
]
ordering = ["-start_date_normalized", "-created_at"]
search_fields = [
"organisation_name",
"registration_number",
@@ -829,7 +872,10 @@ class InspectionViewSet(ReadOnlyModelViewSet):
operation_description=(
"Возвращает список проверок из Единого реестра.\n"
"Поддерживает фильтрацию по: inn, ogrn, registration_number, "
"is_federal_law_248, data_year, data_month, load_batch.\n"
"is_federal_law_248, data_year, data_month, load_batch, "
"date_from, date_to.\n"
"Поддерживает сортировку по: start_date_normalized, "
"end_date_normalized, created_at, updated_at.\n"
"Поддерживает поиск по: organisation_name, registration_number, "
"inn, ogrn, control_authority.\n"
f"{REGISTRY_ORGANIZATION_SEARCH_DESCRIPTION}"
@@ -2202,7 +2248,7 @@ def _native_field_map(source: str) -> dict[str, str]:
"external_id": "registration_number",
"organisation_name": "organisation_name",
"title": "control_authority",
"record_date": "start_date",
"record_date": "start_date_normalized",
"status": "status",
}
@@ -2273,6 +2319,19 @@ def _apply_native_search(queryset, source: str, search: str):
)
def _apply_record_date_range(queryset, record_date_field: str, params: dict):
"""Filter normalized or ISO-compatible record dates inclusively."""
date_from = params.get("date_from")
if date_from:
queryset = queryset.filter(
**{f"{record_date_field}__gte": date_from.isoformat()}
)
date_to = params.get("date_to")
if date_to:
queryset = queryset.filter(**{f"{record_date_field}__lte": date_to.isoformat()})
return queryset
def _route_model_sources(descriptor) -> set[str]:
if descriptor.source == ParserLoadLog.Source.TRUDVSEM:
return set(VACANCY_RECORD_SOURCES)
@@ -2317,6 +2376,8 @@ def _filter_native_result_queryset(source: str, params: dict, sources: set[str])
queryset = queryset.filter(**{model_field: value})
if params.get("record_date") and field_map.get("record_date"):
queryset = queryset.filter(**{field_map["record_date"]: params["record_date"]})
if field_map.get("record_date"):
queryset = _apply_record_date_range(queryset, field_map["record_date"], params)
if params.get("search"):
queryset = _apply_native_search(queryset, source, params["search"])
ordering = _safe_ordering(
@@ -2356,6 +2417,7 @@ def _filter_generic_result_queryset(
queryset = queryset.filter(**{field: value})
if params.get("record_date"):
queryset = queryset.filter(record_date=params["record_date"])
queryset = _apply_record_date_range(queryset, "record_date", params)
if params.get("search"):
search = params["search"]
queryset = queryset.filter(

2
tests/apps/organizations/test_test_companies_commands.py
View File

@@ -133,7 +133,7 @@ class TestCompaniesCommandsTest(TestCase):
self.assertEqual(OrganizationSourceRecord.objects.count(), 20 * 15)
@override_settings(STATE_CORP_EXCHANGE_TOKEN=TEST_EXCHANGE_TOKEN)
def test_created_source_records_are_exported_to_state_corp_package(self):
def test_created_source_records_are_included_in_state_corp_package(self):
call_command("create_test_companies", stdout=StringIO())
company_inns = list(

27
tests/apps/parsers/test_checko_collection.py
View File

@@ -89,4 +89,29 @@ class CheckoCollectionClaimTest(TestCase):
checko_source=CheckoCollectionAttempt.Source.INSPECTIONS,
)
self.assertEqual([target.organization_id for target in targets], [str(second.id)])
self.assertEqual(
[target.organization_id for target in targets], [str(second.id)]
)
def test_lookup_targets_include_corporation_members_outside_opk(self):
self.organization.gk_name = 'Госкорпорация "Росатом"'
self.organization.save(update_fields=["gk_name"])
outside_scope = create_directory_organization(
pn_name='ООО "Вне контура обмена"',
mn_ogrn=1027700000399,
mn_inn=7701000399,
in_kpp=770101001,
mn_okpo="11223344",
)
targets = parser_tasks._active_registry_lookup_targets(
organization_ids=[
str(self.organization.id),
str(outside_scope.id),
],
)
self.assertEqual(
[target.organization_id for target in targets],
[str(self.organization.id)],
)

20
tests/apps/parsers/test_tasks.py
View File

@@ -1307,6 +1307,26 @@ class FNSApiSyncTaskTestCase(TestCase):
self.assertEqual(targets, ["1000000001", "1000000002"])
def test_active_registry_targets_include_corporation_members_outside_opk(self):
OrganizationFactory(
inn="1000000010",
gk_name='Госкорпорация "Роскосмос"',
opk_registry_membership=False,
goz_participation=False,
ropk_num="",
)
OrganizationFactory(
inn="1000000011",
gk_name="",
opk_registry_membership=False,
goz_participation=False,
ropk_num="",
)
targets = parser_tasks._active_fns_registry_inns(limit=10)
self.assertEqual(targets, ["1000000010"])
@override_settings(
CELERY_TASK_ALWAYS_EAGER=True,

92
tests/apps/parsers/test_views.py
View File

@@ -8,6 +8,7 @@ import io
import os
import tempfile
import zipfile
from datetime import date
from unittest.mock import Mock, patch
from apps.parsers.models import (
@@ -480,6 +481,97 @@ class ParsersViewSetTest(APITestCase):
self.assertEqual(response.data["data"][0]["id"], hh_record.id)
self.assertEqual(response.data["data"][0]["source"], "hh")
def test_source_results_filter_and_sort_by_record_date(self):
arbitration_older = GenericParserRecord.objects.create(
load_batch=1,
source=ParserLoadLog.Source.ARBITRATION,
external_id="arbitration-older",
record_date="2024-01-15",
)
arbitration_newer = GenericParserRecord.objects.create(
load_batch=1,
source=ParserLoadLog.Source.ARBITRATION,
external_id="arbitration-newer",
record_date="2024-03-15",
)
vacancy_older = GenericParserRecord.objects.create(
load_batch=1,
source="hh",
external_id="hh-older",
record_date="2024-02-01",
payload={"vacancy_source": "hh"},
)
vacancy_newer = GenericParserRecord.objects.create(
load_batch=1,
source="hh",
external_id="hh-newer",
record_date="2024-04-01",
payload={"vacancy_source": "hh"},
)
self.client.force_authenticate(self.user)
arbitration_response = self.client.get(
"/api/v1/arbitration/cases/",
{"date_from": "2024-02-01", "ordering": "-record_date"},
)
vacancy_response = self.client.get(
"/api/v1/trudvsem/vacancies/",
{"date_to": "2024-02-15", "ordering": "record_date"},
)
self.assertEqual(arbitration_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in arbitration_response.data["data"]],
[arbitration_newer.id],
)
self.assertNotIn(
arbitration_older.id,
[item["id"] for item in arbitration_response.data["data"]],
)
self.assertEqual(vacancy_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in vacancy_response.data["data"]],
[vacancy_older.id],
)
self.assertNotIn(
vacancy_newer.id,
[item["id"] for item in vacancy_response.data["data"]],
)
def test_inspection_source_results_filter_and_sort_by_record_date(self):
older = InspectionRecordFactory(
start_date="2024-01-10",
start_date_normalized=date(2024, 1, 10),
)
newer = InspectionRecordFactory(
start_date="2024-03-10",
start_date_normalized=date(2024, 3, 10),
)
self.client.force_authenticate(self.user)
response = self.client.get(
"/api/v1/proverki/",
{"date_from": "2024-02-01", "ordering": "-start_date_normalized"},
)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in response.data["data"]],
[newer.id],
)
self.assertNotIn(older.id, [item["id"] for item in response.data["data"]])
source_response = self.client.get(
"/api/v1/parsers/results/inspections/",
{"date_from": "2024-02-01", "ordering": "-record_date"},
)
self.assertEqual(source_response.status_code, status.HTTP_200_OK)
self.assertEqual(
[item["id"] for item in source_response.data["data"]],
[newer.id],
)
def test_parser_results_v1_enrich_missing_organization_fields_without_contract_change(
self,
):