diff --git a/src/apps/parsers/fssp_csv.py b/src/apps/parsers/fssp_csv.py index b0fa512..d92cb2d 100644 --- a/src/apps/parsers/fssp_csv.py +++ b/src/apps/parsers/fssp_csv.py @@ -84,7 +84,7 @@ def _termination(payload: dict, warnings: set[str]) -> None: if not raw: warnings.add("unparsed_termination") return - match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}", raw) + match = re.search(r"[0-9]{2}\.[0-9]{2}\.[0-9]{4}|[0-9]{4}-[0-9]{2}-[0-9]{2}", raw) if match: payload["terminated_on"] = _date(match.group(), warnings) for label, field in ( @@ -94,6 +94,9 @@ def _termination(payload: dict, warnings: set[str]) -> None: ): match = re.search(rf"\b{label}\.\s*([0-9]+)", raw) if match: + if field == "termination_point" and "п.п." in raw: + warnings.add("unparsed_termination") + continue payload[field] = match.group(1) if not payload["terminated_on"] or not payload["termination_article"]: warnings.add("unparsed_termination") @@ -207,7 +210,7 @@ def iter_fssp_records( number = payload["proceeding_number"] or "" yield SourceRecordInput( external_id=f"{inn}:{number}", - title=number[:255], + title=f"Исполнительное производство {number}"[:255], organization_name=payload["debtor_name"] or "", record_type="enforcement_proceeding", inn=inn, diff --git a/src/apps/parsers/fssp_snapshot.py b/src/apps/parsers/fssp_snapshot.py index 13765c6..4b207c8 100644 --- a/src/apps/parsers/fssp_snapshot.py +++ b/src/apps/parsers/fssp_snapshot.py @@ -59,7 +59,13 @@ def _identity(row, index: OwnOrganizationIndex): return None, "invalid_proceeding_number" if not row.payload["debtor_raw"] or not row.payload["proceeding_raw"]: return None, "missing_fssp_required_field" - return index.resolve(inn=row.inn, ogrn="") + organization, reason = index.resolve(inn=row.inn, ogrn="") + if not reason and ( + not re.fullmatch(r"[0-9]{13}", organization.ogrn or "") + or not re.fullmatch(r"[0-9]{8}", organization.okpo or "") + ): + return organization, "invalid_canonical_requisites" + return organization, reason def _stage_input(item, raw_artifact, aggregate, index, counts, on_progress): diff --git a/src/apps/parsers/migrations/0043_fssp_disabled_schedule.py b/src/apps/parsers/migrations/0043_fssp_disabled_schedule.py new file mode 100644 index 0000000..6d5a4a9 --- /dev/null +++ b/src/apps/parsers/migrations/0043_fssp_disabled_schedule.py @@ -0,0 +1,34 @@ +"""Daily FSSP collection remains disabled until complete dev acceptance.""" + +from django.db import migrations + + +def seed(apps, schema_editor): + intervals = apps.get_model("django_celery_beat", "IntervalSchedule") + tasks = apps.get_model("django_celery_beat", "PeriodicTask") + interval, _ = intervals.objects.get_or_create(every=1, period="days") + tasks.objects.update_or_create( + name="parser:fssp-enforcement-proceedings:daily", + defaults={ + "task": "parsers.fssp_enforcement_proceedings.refresh", + "interval": interval, + "enabled": False, + "args": "[]", + "kwargs": "{}", + "description": "Enable only after a complete successful snapshot publication on dev.", + }, + ) + + +def unseed(apps, schema_editor): + apps.get_model("django_celery_beat", "PeriodicTask").objects.filter( + name="parser:fssp-enforcement-proceedings:daily" + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("parsers", "0042_fssp_sources"), + ("django_celery_beat", "0018_improve_crontab_helptext"), + ] + operations = [migrations.RunPython(seed, unseed)] diff --git a/src/apps/parsers/snapshot_metadata.py b/src/apps/parsers/snapshot_metadata.py index 2777220..c70282f 100644 --- a/src/apps/parsers/snapshot_metadata.py +++ b/src/apps/parsers/snapshot_metadata.py @@ -1,7 +1,7 @@ """Public snapshot provenance shared by cards, dashboard and parser history.""" from apps.parsers.models import ParserSourceArtifact, ParserStagedRecord -from django.db.models import Count +from django.db.models import Count, Q def attach_snapshot_artifacts(logs) -> None: @@ -10,10 +10,17 @@ def attach_snapshot_artifacts(logs) -> None: if not logs: return artifacts = {} - for artifact in ParserSourceArtifact.objects.filter( - source__in={log.source for log in logs}, - load_batch__in={log.batch_id for log in logs}, - ).order_by("-created_at", "-uid"): + for artifact in ( + ParserSourceArtifact.objects.filter( + source__in={log.source for log in logs}, + load_batch__in={log.batch_id for log in logs}, + ) + .exclude( + Q(source="fssp_enforcement_proceedings") + & Q(metadata__dataset__isnull=False) + ) + .order_by("-created_at", "-uid") + ): artifacts.setdefault((artifact.source, artifact.load_batch), artifact) legacy_artifacts = [] for artifact in artifacts.values(): diff --git a/src/apps/parsers/source_cards.py b/src/apps/parsers/source_cards.py index 53668a2..d8cc1db 100644 --- a/src/apps/parsers/source_cards.py +++ b/src/apps/parsers/source_cards.py @@ -46,12 +46,14 @@ STALE_PENDING_MAX_AGE_MINUTES = 24 * 60 SOURCE_CARD_STATS_CACHE_TIMEOUT_SECONDS = 7 * 24 * 60 * 60 SOURCE_CARD_STATS_CACHE_SCHEMA_VERSION = 4 SNAPSHOT_CARD_SLUGS = { + "fssp_enforcement_proceedings": "fssp-enforcement-proceedings", "budget_ubpandnubp": "budget-process-registry", "fns_sme_support_recipients": "sme-support-recipients-registry", "sro_membership_check": "sro-membership-check", "fedresurs_company_profile": "fedresurs-company-profile", } SNAPSHOT_SOURCE_GROUPS = ( + "enforcement_proceedings", "budget_process_registry", "government_support", "sro_membership", @@ -503,6 +505,23 @@ SOURCE_CARD_DEFINITIONS: tuple[SourceCardDefinition, ...] = ( ), refresh_interval=timedelta(days=1), ), + SourceCardDefinition( + slug="fssp-enforcement-proceedings", + title="ФССП: Банк данных исполнительных производств", + description="Открытые CSV ФССП для юридических лиц.", + order=1000, + task_names=("parsers.fssp_enforcement_proceedings.refresh",), + source_items=( + SourceItemDefinition( + code="fssp_inn_search", + title="Исполнительные производства ФССП", + description="Производства организаций канонического реестра.", + parser_source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS, + refresh_key="fssp_enforcement_proceedings", + ), + ), + refresh_interval=timedelta(days=1), + ), ) SOURCE_CARD_BY_SLUG = {item.slug: item for item in SOURCE_CARD_DEFINITIONS} @@ -1328,11 +1347,20 @@ class SourceCardService: from apps.parsers.tasks_registry_snapshots import ( parse_budget_registry, parse_fedresurs_company_profiles, + parse_fssp_enforcement_proceedings, parse_sme_support_recipients, parse_sro_membership, ) - if definition.slug == "budget-process-registry": + if definition.slug == "fssp-enforcement-proceedings": + task_specs = ( + ( + parse_fssp_enforcement_proceedings, + "parsers.fssp_enforcement_proceedings.refresh", + "fssp_enforcement_proceedings", + ), + ) + elif definition.slug == "budget-process-registry": task_specs = ( ( parse_budget_registry, diff --git a/src/apps/parsers/source_registry.py b/src/apps/parsers/source_registry.py index 2c20bae..2044cee 100644 --- a/src/apps/parsers/source_registry.py +++ b/src/apps/parsers/source_registry.py @@ -56,6 +56,19 @@ class ParserSourceDescriptor: PARSER_SOURCES: dict[str, ParserSourceDescriptor] = { + "fssp_enforcement_proceedings": ParserSourceDescriptor( + key="fssp_enforcement_proceedings", + source=ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS, + title="ФССП: Банк данных исполнительных производств", + agency="ФССП России", + data_scope="Исполнительные производства организаций канонического реестра", + task_name="parsers.fssp_enforcement_proceedings.refresh", + mode="open_data_csv", + upstream_url="https://opendata.fssp.gov.ru/", + access_method="open_data", + parser_strategy="streaming_two_dataset_snapshot", + admin_only=True, + ), "fedresurs_company_profile": ParserSourceDescriptor( key="fedresurs_company_profile", source=ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, diff --git a/src/apps/parsers/tasks.py b/src/apps/parsers/tasks.py index 49214bf..bf7e13d 100644 --- a/src/apps/parsers/tasks.py +++ b/src/apps/parsers/tasks.py @@ -75,6 +75,7 @@ from apps.parsers.source_registry import PARSER_SOURCES from apps.parsers.tasks_registry_snapshots import ( # noqa: F401 parse_budget_registry, parse_fedresurs_company_profiles, + parse_fssp_enforcement_proceedings, parse_sme_support_recipients, parse_sro_membership, ) diff --git a/src/apps/parsers/tasks_registry_snapshots.py b/src/apps/parsers/tasks_registry_snapshots.py index 8ea366d..b2af88d 100644 --- a/src/apps/parsers/tasks_registry_snapshots.py +++ b/src/apps/parsers/tasks_registry_snapshots.py @@ -62,7 +62,7 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict: "version": artifact.version or None, "checksum_sha256": artifact.sha256 or None, } - if artifact.source == "sro_membership_check": + if artifact.source in ("sro_membership_check", "fssp_enforcement_proceedings"): payload.update(artifact.metadata) job.meta = { **job.meta, @@ -79,7 +79,11 @@ def _complete_snapshot_job(job, load_log, artifact, result) -> dict: def _claim_snapshot_job(job, source, task_name): with transaction.atomic(): - if source in ("sro_membership_check", "fedresurs_company_profile"): + if source in ( + "sro_membership_check", + "fedresurs_company_profile", + "fssp_enforcement_proceedings", + ): sequence, _ = ParserBatchSequence.objects.get_or_create(source=source) ParserBatchSequence.objects.select_for_update().get(pk=sequence.pk) if ( @@ -107,6 +111,18 @@ def _update_snapshot_progress(job, queried, candidates, found, quarantined): job.refresh_from_db() if job.status != JobStatus.STARTED: raise SnapshotValidationError("snapshot_job_no_longer_active") + if job.meta.get("source") == "fssp_enforcement_proceedings": + job.update_progress(5, f"Прочитано {queried} строк CSV") + BackgroundJob.objects.filter(pk=job.pk, status=JobStatus.STARTED).update( + meta={ + **job.meta, + "raw_records_count": queried, + "matched_records_count": found, + "quarantined_records_count": quarantined, + }, + updated_at=timezone.now(), + ) + return if queried % 10 == 0 or queried == candidates: job.update_progress( 5 + (90 * queried // max(candidates, 1)), @@ -290,3 +306,23 @@ def parse_fedresurs_company_profiles( track_progress=True, mode=mode, ) + + +@shared_task( + bind=True, + name="parsers.fssp_enforcement_proceedings.refresh", + soft_time_limit=8 * 3600, + time_limit=8 * 3600 + 300, +) +def parse_fssp_enforcement_proceedings( + self, *, requested_by_id: int | None = None +) -> dict: + from apps.parsers.fssp_snapshot import refresh_fssp_enforcement_proceedings + + return _run_snapshot( + self, + source="fssp_enforcement_proceedings", + refresh=refresh_fssp_enforcement_proceedings, + requested_by_id=requested_by_id, + track_progress=True, + ) diff --git a/src/apps/parsers/views.py b/src/apps/parsers/views.py index 72b3d39..afe3838 100644 --- a/src/apps/parsers/views.py +++ b/src/apps/parsers/views.py @@ -146,6 +146,7 @@ SOURCE_RESULT_TAGS = { } TASKS_BY_NAME = { + "parsers.fssp_enforcement_proceedings.refresh": tasks.parse_fssp_enforcement_proceedings, "parsers.fedresurs_company_profile.refresh": tasks.parse_fedresurs_company_profiles, "apps.parsers.tasks.parse_industrial_production": tasks.parse_industrial_production, "apps.parsers.tasks.parse_industrial_products": tasks.parse_industrial_products, diff --git a/src/core/openapi.json b/src/core/openapi.json index ad0750f..0aecd63 100644 --- a/src/core/openapi.json +++ b/src/core/openapi.json @@ -9597,7 +9597,8 @@ "sro_membership", "budget_process_registry", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "enforcement_proceedings" ] } }, @@ -9670,7 +9671,7 @@ { "name": "ordering", "in": "query", - "description": "Сортировка по полям: record_date, record_type, external_id, organization__name, extension__organization__name, extension__organization__full_name, created_at, updated_at, title, uid, extension__organization__inn, extension__organization__ogrn, extension__organization__okpo, status, payload__attestation_status, payload__full_name, payload__medo_address, payload__registration_number, payload__uk_hm_treasury, payload__european_union, payload__united_states, payload__switzerland, payload__united_states_sectoral, payload__uk_uksl, payload__ukraine, payload__sentiment, payload__news_source, events_count, documents_count, first_event_date, organization__full_name, organization__inn, organization__ogrn, organization__okpo, payload__classification__organization_type__name, payload__classification__establishment_kind__name, payload__budget__level__name, payload__address__region__name, payload__is_separate_division, payload__support_form__name, payload__support_kind__name, payload__support_until, payload__provider__name, payload__sme_category__name, payload__has_violation, amount, payload__membership_status, payload__region, payload__sro_name, payload__admission_date, payload__status__name, payload__executive_body_name, payload__registration_date, payload__publication_count, payload__reorganization_count, payload__captured_at. Для обратной сортировки используйте префикс -. Несколько полей перечисляются через запятую без пробелов, например -record_date,-external_id. Значения record_date с null сортируются последними.", + "description": "Сортировка по полям: record_date, record_type, external_id, organization__name, extension__organization__name, extension__organization__full_name, created_at, updated_at, title, uid, extension__organization__inn, extension__organization__ogrn, extension__organization__okpo, status, payload__attestation_status, payload__full_name, payload__medo_address, payload__registration_number, payload__uk_hm_treasury, payload__european_union, payload__united_states, payload__switzerland, payload__united_states_sectoral, payload__uk_uksl, payload__ukraine, payload__sentiment, payload__news_source, events_count, documents_count, first_event_date, payload__document_raw, payload__terminated_on, payload__service_text, organization__full_name, organization__inn, organization__ogrn, organization__okpo, payload__classification__organization_type__name, payload__classification__establishment_kind__name, payload__budget__level__name, payload__address__region__name, payload__is_separate_division, payload__support_form__name, payload__support_kind__name, payload__support_until, payload__provider__name, payload__sme_category__name, payload__has_violation, amount, payload__membership_status, payload__region, payload__sro_name, payload__admission_date, payload__status__name, payload__executive_body_name, payload__registration_date, payload__publication_count, payload__reorganization_count, payload__captured_at. Для обратной сортировки используйте префикс -. Несколько полей перечисляются через запятую без пробелов, например -record_date,-external_id. Значения record_date с null сортируются последними.", "required": false, "schema": { "anyOf": [ @@ -10269,7 +10270,8 @@ "sro_membership", "budget_process_registry", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "enforcement_proceedings" ] } }, @@ -10589,6 +10591,15 @@ "schema": { "type": "boolean" } + }, + { + "name": "has_enforcement_proceedings", + "in": "query", + "description": "Фильтр наличия группы источников enforcement_proceedings.", + "required": false, + "schema": { + "type": "boolean" + } } ], "responses": { @@ -12658,7 +12669,8 @@ "budget_ubpandnubp", "sro_membership_check", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "fssp_enforcement_proceedings" ] }, "title": { @@ -12939,7 +12951,8 @@ "budget_ubpandnubp", "sro_membership_check", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "fssp_enforcement_proceedings" ], "readOnly": true }, @@ -13779,7 +13792,8 @@ "budget_ubpandnubp", "sro_membership_check", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "fssp_enforcement_proceedings" ], "readOnly": true }, @@ -14410,6 +14424,7 @@ "sro_membership_check", "integrated_structure_documents", "fedresurs_company_profile", + "fssp_enforcement_proceedings", "financial-indicators", "public-procurements", "manufacturers-and-products", @@ -14426,7 +14441,8 @@ "budget-process-registry", "sro-membership-check", "integrated-structure-documents", - "fedresurs-company-profile" + "fedresurs-company-profile", + "fssp-enforcement-proceedings" ], "readOnly": true }, @@ -18136,6 +18152,9 @@ { "$ref": "#/components/schemas/SroMembershipRecordList" }, + { + "$ref": "#/components/schemas/FsspEnforcementProceedingRecordList" + }, { "$ref": "#/components/schemas/IntegratedStructureRecordList" }, @@ -18161,6 +18180,7 @@ "budget_process_registry": "#/components/schemas/BudgetRegistryRecordList", "government_support": "#/components/schemas/SmeSupportRecordList", "sro_membership": "#/components/schemas/SroMembershipRecordList", + "enforcement_proceedings": "#/components/schemas/FsspEnforcementProceedingRecordList", "integrated_structure_documents": "#/components/schemas/IntegratedStructureRecordList", "fedresurs_company_profile": "#/components/schemas/FedresursCompanyProfileRecordList" } @@ -18394,7 +18414,8 @@ "sro_membership", "budget_process_registry", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "enforcement_proceedings" ] } }, @@ -18874,6 +18895,9 @@ { "$ref": "#/components/schemas/SroMembershipRecordDetail" }, + { + "$ref": "#/components/schemas/FsspEnforcementProceedingRecordDetail" + }, { "$ref": "#/components/schemas/IntegratedStructureRecordDetail" }, @@ -18899,6 +18923,7 @@ "budget_process_registry": "#/components/schemas/BudgetRegistryRecordDetail", "government_support": "#/components/schemas/SmeSupportRecordDetail", "sro_membership": "#/components/schemas/SroMembershipRecordDetail", + "enforcement_proceedings": "#/components/schemas/FsspEnforcementProceedingRecordDetail", "integrated_structure_documents": "#/components/schemas/IntegratedStructureRecordDetail", "fedresurs_company_profile": "#/components/schemas/FedresursCompanyProfileRecordDetail" } @@ -18934,7 +18959,8 @@ "sro_membership", "budget_process_registry", "integrated_structure_documents", - "fedresurs_company_profile" + "fedresurs_company_profile", + "enforcement_proceedings" ], "readOnly": true }, @@ -19652,6 +19678,135 @@ "full_name" ] }, + "FsspSourceRecordOrganization": { + "type": "object", + "properties": { + "uid": { + "title": "Uid", + "type": "string", + "format": "uuid", + "readOnly": true + }, + "name": { + "title": "Name", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "full_name": { + "title": "Full name", + "type": "string", + "readOnly": true, + "nullable": true + }, + "short_name": { + "title": "Short name", + "type": "string", + "readOnly": true, + "nullable": true + }, + "normalized_name": { + "title": "Normalized name", + "type": "string", + "readOnly": true, + "nullable": true + }, + "inn": { + "title": "Inn", + "type": "string", + "readOnly": true, + "minLength": 1, + "pattern": "^[0-9]{10}$" + }, + "kpp": { + "title": "Kpp", + "type": "string", + "readOnly": true, + "nullable": true + }, + "ogrn": { + "title": "Ogrn", + "type": "string", + "readOnly": true, + "minLength": 1, + "pattern": "^[0-9]{13}$" + }, + "okpo": { + "title": "Okpo", + "type": "string", + "readOnly": true, + "minLength": 1, + "pattern": "^[0-9]{8}$" + }, + "legal_address": { + "title": "Legal address", + "type": "string", + "readOnly": true, + "nullable": true + }, + "business_activity": { + "title": "Business activity", + "type": "string", + "readOnly": true, + "nullable": true + }, + "dep": { + "title": "Dep", + "type": "string", + "readOnly": true, + "nullable": true + }, + "integrated_structure": { + "title": "Integrated structure", + "type": "string", + "readOnly": true, + "nullable": true + }, + "gk_name": { + "title": "Gk name", + "type": "string", + "readOnly": true, + "nullable": true + }, + "in_korp_name": { + "title": "In korp name", + "type": "string", + "readOnly": true, + "nullable": true + }, + "state_sector_name": { + "title": "State sector name", + "type": "string", + "readOnly": true, + "nullable": true + }, + "filial": { + "title": "Filial", + "type": "string", + "readOnly": true, + "nullable": true + }, + "is_branch": { + "title": "Is branch", + "type": "boolean", + "readOnly": true, + "nullable": true + }, + "ogrip": { + "title": "Ogrip", + "type": "string", + "readOnly": true, + "nullable": true + } + }, + "required": [ + "uid", + "name", + "inn", + "ogrn", + "okpo" + ] + }, "LegacyOrganizationSourceRecordList": { "type": "object", "properties": { @@ -20741,6 +20896,12 @@ "-documents_count", "first_event_date", "-first_event_date", + "payload__document_raw", + "-payload__document_raw", + "payload__terminated_on", + "-payload__terminated_on", + "payload__service_text", + "-payload__service_text", "organization__full_name", "-organization__full_name", "organization__inn", @@ -20797,7 +20958,7 @@ }, "SourceRecordOrderingSequence": { "type": "string", - "pattern": "^(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at)(?:,(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at))+$", + "pattern": "^(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|payload__document_raw|-payload__document_raw|payload__terminated_on|-payload__terminated_on|payload__service_text|-payload__service_text|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at)(?:,(?:record_date|-record_date|record_type|-record_type|external_id|-external_id|organization__name|-organization__name|extension__organization__name|-extension__organization__name|extension__organization__full_name|-extension__organization__full_name|created_at|-created_at|updated_at|-updated_at|title|-title|uid|-uid|extension__organization__inn|-extension__organization__inn|extension__organization__ogrn|-extension__organization__ogrn|extension__organization__okpo|-extension__organization__okpo|status|-status|payload__attestation_status|-payload__attestation_status|payload__full_name|-payload__full_name|payload__medo_address|-payload__medo_address|payload__registration_number|-payload__registration_number|payload__uk_hm_treasury|-payload__uk_hm_treasury|payload__european_union|-payload__european_union|payload__united_states|-payload__united_states|payload__switzerland|-payload__switzerland|payload__united_states_sectoral|-payload__united_states_sectoral|payload__uk_uksl|-payload__uk_uksl|payload__ukraine|-payload__ukraine|payload__sentiment|-payload__sentiment|payload__news_source|-payload__news_source|events_count|-events_count|documents_count|-documents_count|first_event_date|-first_event_date|payload__document_raw|-payload__document_raw|payload__terminated_on|-payload__terminated_on|payload__service_text|-payload__service_text|organization__full_name|-organization__full_name|organization__inn|-organization__inn|organization__ogrn|-organization__ogrn|organization__okpo|-organization__okpo|payload__classification__organization_type__name|-payload__classification__organization_type__name|payload__classification__establishment_kind__name|-payload__classification__establishment_kind__name|payload__budget__level__name|-payload__budget__level__name|payload__address__region__name|-payload__address__region__name|payload__is_separate_division|-payload__is_separate_division|payload__support_form__name|-payload__support_form__name|payload__support_kind__name|-payload__support_kind__name|payload__support_until|-payload__support_until|payload__provider__name|-payload__provider__name|payload__sme_category__name|-payload__sme_category__name|payload__has_violation|-payload__has_violation|amount|-amount|payload__membership_status|-payload__membership_status|payload__region|-payload__region|payload__sro_name|-payload__sro_name|payload__admission_date|-payload__admission_date|payload__status__name|-payload__status__name|payload__executive_body_name|-payload__executive_body_name|payload__registration_date|-payload__registration_date|payload__publication_count|-payload__publication_count|payload__reorganization_count|-payload__reorganization_count|payload__captured_at|-payload__captured_at))+$", "description": "Two or more allowlisted ordering fields, separated by commas without spaces." }, "AttachmentUnavailableError": { @@ -21263,6 +21424,782 @@ "lineage" ] }, + "FsspEnforcementProceedingListPayload": { + "type": "object", + "properties": { + "queried_inn": { + "type": "string", + "nullable": true + }, + "debtor_name": { + "type": "string", + "nullable": true + }, + "debtor_legal_address": { + "type": "string", + "nullable": true + }, + "debtor_actual_address": { + "type": "string", + "nullable": true + }, + "consolidated_proceeding_number": { + "type": "string", + "nullable": true + }, + "document_raw": { + "type": "string", + "nullable": true + }, + "document_type": { + "type": "string", + "nullable": true + }, + "document_number": { + "type": "string", + "nullable": true + }, + "document_issuer": { + "type": "string", + "nullable": true + }, + "document_subject_raw": { + "type": "string", + "nullable": true + }, + "creditor_inn": { + "type": "string", + "nullable": true + }, + "termination_raw": { + "type": "string", + "nullable": true + }, + "termination_reason": { + "type": "string", + "nullable": true + }, + "termination_article": { + "type": "string", + "nullable": true + }, + "termination_part": { + "type": "string", + "nullable": true + }, + "termination_point": { + "type": "string", + "nullable": true + }, + "service_text": { + "type": "string", + "nullable": true + }, + "subject_raw": { + "type": "string", + "nullable": true + }, + "subject": { + "type": "string", + "nullable": true + }, + "debt_amount": { + "type": "string", + "nullable": true, + "pattern": "^-?[0-9]+\\.[0-9]{2}$" + }, + "claim_amount": { + "type": "string", + "nullable": true, + "pattern": "^[0-9]+\\.[0-9]{2}$" + }, + "debt_currency": { + "type": "string", + "nullable": true + }, + "department_raw": { + "type": "string", + "nullable": true + }, + "department_name": { + "type": "string", + "nullable": true + }, + "department_address": { + "type": "string", + "nullable": true + }, + "bailiff_raw": { + "type": "string", + "nullable": true + }, + "bailiff_name": { + "type": "string", + "nullable": true + }, + "debtor_raw": { + "type": "string" + }, + "debtor_inn": { + "type": "string" + }, + "proceeding_raw": { + "type": "string" + }, + "proceeding_number": { + "type": "string" + }, + "source_url": { + "type": "string" + }, + "dataset": { + "type": "string" + }, + "initiated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "document_date": { + "type": "string", + "format": "date", + "nullable": true + }, + "terminated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "origin_kind": { + "type": "string", + "enum": [ + "web_search", + "open_data_csv" + ], + "nullable": true + }, + "fetched_at": { + "type": "string", + "format": "date-time" + }, + "service_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "label", + "url" + ] + } + }, + "bailiff_phones": { + "type": "array", + "items": { + "type": "string" + } + }, + "parse_warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "raw_columns": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "source_row_number": { + "type": "integer" + }, + "provenance": { + "type": "object", + "additionalProperties": true + }, + "observations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + } + }, + "required": [ + "queried_inn", + "debtor_name", + "debtor_legal_address", + "debtor_actual_address", + "consolidated_proceeding_number", + "document_raw", + "document_type", + "document_number", + "document_issuer", + "document_subject_raw", + "creditor_inn", + "termination_raw", + "termination_reason", + "termination_article", + "termination_part", + "termination_point", + "service_text", + "subject_raw", + "subject", + "debt_amount", + "claim_amount", + "debt_currency", + "department_raw", + "department_name", + "department_address", + "bailiff_raw", + "bailiff_name", + "debtor_raw", + "debtor_inn", + "proceeding_raw", + "proceeding_number", + "source_url", + "dataset", + "initiated_on", + "document_date", + "terminated_on", + "origin_kind", + "fetched_at", + "service_links", + "bailiff_phones", + "parse_warnings", + "raw_columns", + "source_row_number" + ] + }, + "FsspEnforcementProceedingDetailPayload": { + "type": "object", + "properties": { + "queried_inn": { + "type": "string", + "nullable": true + }, + "debtor_name": { + "type": "string", + "nullable": true + }, + "debtor_legal_address": { + "type": "string", + "nullable": true + }, + "debtor_actual_address": { + "type": "string", + "nullable": true + }, + "consolidated_proceeding_number": { + "type": "string", + "nullable": true + }, + "document_raw": { + "type": "string", + "nullable": true + }, + "document_type": { + "type": "string", + "nullable": true + }, + "document_number": { + "type": "string", + "nullable": true + }, + "document_issuer": { + "type": "string", + "nullable": true + }, + "document_subject_raw": { + "type": "string", + "nullable": true + }, + "creditor_inn": { + "type": "string", + "nullable": true + }, + "termination_raw": { + "type": "string", + "nullable": true + }, + "termination_reason": { + "type": "string", + "nullable": true + }, + "termination_article": { + "type": "string", + "nullable": true + }, + "termination_part": { + "type": "string", + "nullable": true + }, + "termination_point": { + "type": "string", + "nullable": true + }, + "service_text": { + "type": "string", + "nullable": true + }, + "subject_raw": { + "type": "string", + "nullable": true + }, + "subject": { + "type": "string", + "nullable": true + }, + "debt_amount": { + "type": "string", + "nullable": true, + "pattern": "^-?[0-9]+\\.[0-9]{2}$" + }, + "claim_amount": { + "type": "string", + "nullable": true, + "pattern": "^[0-9]+\\.[0-9]{2}$" + }, + "debt_currency": { + "type": "string", + "nullable": true + }, + "department_raw": { + "type": "string", + "nullable": true + }, + "department_name": { + "type": "string", + "nullable": true + }, + "department_address": { + "type": "string", + "nullable": true + }, + "bailiff_raw": { + "type": "string", + "nullable": true + }, + "bailiff_name": { + "type": "string", + "nullable": true + }, + "debtor_raw": { + "type": "string" + }, + "debtor_inn": { + "type": "string" + }, + "proceeding_raw": { + "type": "string" + }, + "proceeding_number": { + "type": "string" + }, + "source_url": { + "type": "string" + }, + "dataset": { + "type": "string" + }, + "initiated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "document_date": { + "type": "string", + "format": "date", + "nullable": true + }, + "terminated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "origin_kind": { + "type": "string", + "enum": [ + "web_search", + "open_data_csv" + ], + "nullable": true + }, + "fetched_at": { + "type": "string", + "format": "date-time" + }, + "service_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "label", + "url" + ] + } + }, + "bailiff_phones": { + "type": "array", + "items": { + "type": "string" + } + }, + "parse_warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "raw_columns": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "source_row_number": { + "type": "integer" + }, + "provenance": { + "type": "object", + "additionalProperties": true + }, + "observations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + } + }, + "required": [ + "queried_inn", + "debtor_name", + "debtor_legal_address", + "debtor_actual_address", + "consolidated_proceeding_number", + "document_raw", + "document_type", + "document_number", + "document_issuer", + "document_subject_raw", + "creditor_inn", + "termination_raw", + "termination_reason", + "termination_article", + "termination_part", + "termination_point", + "service_text", + "subject_raw", + "subject", + "debt_amount", + "claim_amount", + "debt_currency", + "department_raw", + "department_name", + "department_address", + "bailiff_raw", + "bailiff_name", + "debtor_raw", + "debtor_inn", + "proceeding_raw", + "proceeding_number", + "source_url", + "dataset", + "initiated_on", + "document_date", + "terminated_on", + "origin_kind", + "fetched_at", + "service_links", + "bailiff_phones", + "parse_warnings", + "raw_columns", + "source_row_number" + ] + }, + "FsspEnforcementProceedingRecordList": { + "type": "object", + "properties": { + "uid": { + "title": "UID", + "type": "string", + "format": "uuid", + "readOnly": true + }, + "extension_uid": { + "title": "Extension uid", + "type": "string", + "format": "uuid", + "readOnly": true + }, + "source_group": { + "type": "string", + "enum": [ + "enforcement_proceedings" + ] + }, + "record_type": { + "type": "string", + "enum": [ + "enforcement_proceeding" + ] + }, + "source": { + "type": "string", + "enum": [ + "fssp_enforcement_proceedings" + ] + }, + "external_id": { + "title": "Внешний ID", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "title": { + "title": "Заголовок", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "record_date": { + "type": "string", + "format": "date", + "nullable": true + }, + "amount": { + "type": "string", + "pattern": "^-?[0-9]+\\.[0-9]{2}$", + "nullable": true + }, + "status": { + "type": "string", + "enum": [ + "unknown", + "ended", + "terminated" + ] + }, + "url": { + "title": "URL", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "payload": { + "$ref": "#/components/schemas/FsspEnforcementProceedingListPayload" + }, + "legacy_model": { + "type": "string", + "enum": [ + "" + ] + }, + "legacy_pk": { + "type": "string", + "enum": [ + "" + ] + }, + "load_batch": { + "title": "ID пакета загрузки", + "type": "integer", + "readOnly": true, + "nullable": true + }, + "created_at": { + "title": "Дата создания", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "updated_at": { + "title": "Дата обновления", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "financial_lines": { + "type": "array", + "items": {}, + "maxItems": 0 + }, + "organization": { + "$ref": "#/components/schemas/FsspSourceRecordOrganization" + } + }, + "required": [ + "source_group", + "uid", + "source", + "record_type", + "external_id", + "title", + "status", + "created_at", + "updated_at", + "organization", + "payload", + "extension_uid", + "record_date", + "amount", + "url", + "legacy_model", + "legacy_pk", + "load_batch", + "financial_lines" + ] + }, + "FsspEnforcementProceedingRecordDetail": { + "type": "object", + "properties": { + "uid": { + "title": "UID", + "type": "string", + "format": "uuid", + "readOnly": true + }, + "extension_uid": { + "title": "Extension uid", + "type": "string", + "format": "uuid", + "readOnly": true + }, + "source_group": { + "type": "string", + "enum": [ + "enforcement_proceedings" + ] + }, + "record_type": { + "type": "string", + "enum": [ + "enforcement_proceeding" + ] + }, + "source": { + "type": "string", + "enum": [ + "fssp_enforcement_proceedings" + ] + }, + "external_id": { + "title": "Внешний ID", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "title": { + "title": "Заголовок", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "record_date": { + "type": "string", + "format": "date", + "nullable": true + }, + "amount": { + "type": "string", + "pattern": "^-?[0-9]+\\.[0-9]{2}$", + "nullable": true + }, + "status": { + "type": "string", + "enum": [ + "unknown", + "ended", + "terminated" + ] + }, + "url": { + "title": "URL", + "type": "string", + "readOnly": true, + "minLength": 1 + }, + "payload": { + "$ref": "#/components/schemas/FsspEnforcementProceedingDetailPayload" + }, + "legacy_model": { + "type": "string", + "enum": [ + "" + ] + }, + "legacy_pk": { + "type": "string", + "enum": [ + "" + ] + }, + "load_batch": { + "title": "ID пакета загрузки", + "type": "integer", + "readOnly": true, + "nullable": true + }, + "created_at": { + "title": "Дата создания", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "updated_at": { + "title": "Дата обновления", + "type": "string", + "format": "date-time", + "readOnly": true + }, + "financial_lines": { + "type": "array", + "items": {}, + "maxItems": 0 + }, + "organization": { + "$ref": "#/components/schemas/FsspSourceRecordOrganization" + } + }, + "required": [ + "source_group", + "uid", + "source", + "record_type", + "external_id", + "title", + "status", + "created_at", + "updated_at", + "organization", + "payload", + "extension_uid", + "record_date", + "amount", + "url", + "legacy_model", + "legacy_pk", + "load_batch", + "financial_lines" + ] + }, "IntegratedStructureRecordList": { "type": "object", "properties": { diff --git a/src/organizations/fssp_payload_serializers.py b/src/organizations/fssp_payload_serializers.py new file mode 100644 index 0000000..dc81cbf --- /dev/null +++ b/src/organizations/fssp_payload_serializers.py @@ -0,0 +1,60 @@ +"""Public fields of the verified FSSP CSV contract, shared by list and detail.""" + +from rest_framework import serializers + +from organizations.registry_payload_serializers import RegistryUTCDateTimeField + + +class FsspServiceLinkSerializer(serializers.Serializer): + label = serializers.CharField() + url = serializers.CharField() + + +class FsspEnforcementProceedingPayloadSerializer(serializers.Serializer): + origin_kind = serializers.ChoiceField( + choices=("web_search", "open_data_csv"), allow_null=True + ) + queried_inn = serializers.CharField(allow_null=True) + debtor_name = serializers.CharField(allow_null=True) + debtor_legal_address = serializers.CharField(allow_null=True) + debtor_actual_address = serializers.CharField(allow_null=True) + consolidated_proceeding_number = serializers.CharField(allow_null=True) + document_raw = serializers.CharField(allow_null=True) + document_type = serializers.CharField(allow_null=True) + document_number = serializers.CharField(allow_null=True) + document_issuer = serializers.CharField(allow_null=True) + document_subject_raw = serializers.CharField(allow_null=True) + creditor_inn = serializers.CharField(allow_null=True) + termination_raw = serializers.CharField(allow_null=True) + termination_reason = serializers.CharField(allow_null=True) + termination_article = serializers.CharField(allow_null=True) + termination_part = serializers.CharField(allow_null=True) + termination_point = serializers.CharField(allow_null=True) + service_text = serializers.CharField(allow_null=True) + subject_raw = serializers.CharField(allow_null=True) + subject = serializers.CharField(allow_null=True) + debt_amount = serializers.CharField(allow_null=True) + claim_amount = serializers.CharField(allow_null=True) + debt_currency = serializers.CharField(allow_null=True) + department_raw = serializers.CharField(allow_null=True) + department_name = serializers.CharField(allow_null=True) + department_address = serializers.CharField(allow_null=True) + bailiff_raw = serializers.CharField(allow_null=True) + bailiff_name = serializers.CharField(allow_null=True) + debtor_raw = serializers.CharField() + debtor_inn = serializers.CharField() + proceeding_raw = serializers.CharField() + proceeding_number = serializers.CharField() + source_url = serializers.CharField() + initiated_on = serializers.DateField(allow_null=True) + document_date = serializers.DateField(allow_null=True) + terminated_on = serializers.DateField(allow_null=True) + fetched_at = RegistryUTCDateTimeField() + service_links = FsspServiceLinkSerializer(many=True) + bailiff_phones = serializers.ListField(child=serializers.CharField()) + parse_warnings = serializers.ListField(child=serializers.CharField()) + raw_columns = serializers.DictField(child=serializers.CharField()) + dataset = serializers.CharField() + source_row_number = serializers.IntegerField() + provenance = serializers.JSONField(required=False) + observations = serializers.JSONField(required=False) diff --git a/src/organizations/serializers.py b/src/organizations/serializers.py index a306886..1bd39e3 100644 --- a/src/organizations/serializers.py +++ b/src/organizations/serializers.py @@ -322,6 +322,12 @@ class OrganizationSourceRecordSerializer(serializers.ModelSerializer): ) def get_payload(self, obj) -> dict | list | str | int | float | bool | None: # noqa: C901 payload = obj.payload + if obj.source == "fssp_enforcement_proceedings": + from organizations.fssp_payload_serializers import ( + FsspEnforcementProceedingPayloadSerializer, + ) + + return FsspEnforcementProceedingPayloadSerializer(payload).data if obj.source == "fedresurs_company_profile": from organizations.fedresurs_payload import detail_payload diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index abb3f3d..9deac93 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -80,6 +80,7 @@ EXPORT_PROVIDER_NAME_PATTERN = re.compile( ) SNAPSHOT_EXPORT_SOURCES = { + "enforcement_proceedings": "fssp_enforcement_proceedings", "budget_process_registry": "budget_ubpandnubp", "government_support": "fns_sme_support_recipients", "sro_membership": "sro_membership_check", @@ -89,6 +90,7 @@ SNAPSHOT_EXPORT_SOURCES = { GENERATION_ID_PATTERN = re.compile(r"[0-9]{8}T[0-9]{6}Z-[a-f0-9]{8}") SOURCE_GROUP_EXPORT_FILE_STEMS: dict[str, str] = { + SourceGroup.ENFORCEMENT_PROCEEDINGS.value: "fssp-enforcement-proceedings", SourceGroup.SRO_MEMBERSHIP.value: "sro-membership-check", SourceGroup.INTEGRATED_STRUCTURE_DOCUMENTS.value: "integrated-structure-documents", SourceGroup.FEDRESURS_COMPANY_PROFILE.value: "fedresurs-company-profile", diff --git a/src/organizations/source_record_query.py b/src/organizations/source_record_query.py index dfc8d09..480e2f2 100644 --- a/src/organizations/source_record_query.py +++ b/src/organizations/source_record_query.py @@ -9,6 +9,7 @@ from decimal import Decimal, InvalidOperation from uuid import UUID from apps.core.pagination import StandardPagination +from django.core.paginator import Page from django.db.models import ( BooleanField, Case, @@ -23,8 +24,10 @@ from django.db.models import ( from django.db.models.fields.json import KeyTextTransform from django.db.models.functions import Cast, Coalesce, NullIf from drf_yasg import openapi +from rest_framework.exceptions import NotFound REGISTRY_GROUPS = ( + "enforcement_proceedings", "budget_process_registry", "government_support", "sro_membership", @@ -32,6 +35,7 @@ REGISTRY_GROUPS = ( "fedresurs_company_profile", ) REGISTRY_SOURCE_GROUPS = { + "fssp_enforcement_proceedings": "enforcement_proceedings", "budget_ubpandnubp": "budget_process_registry", "fns_sme_support_recipients": "government_support", "sro_membership_check": "sro_membership", @@ -40,6 +44,9 @@ REGISTRY_SOURCE_GROUPS = { } REGISTRY_ORDERING_FIELDS = ( + "payload__document_raw", + "payload__terminated_on", + "payload__service_text", "organization__full_name", "organization__inn", "organization__ogrn", @@ -136,10 +143,30 @@ FILTER_PARAMS = ( class SourceRecordPagination(StandardPagination): page_size = 50 + def paginate_queryset(self, queryset, request, view=None): + try: + return super().paginate_queryset(queryset, request, view) + except NotFound: + if not ( + request.query_params.get("source_group") == "enforcement_proceedings" + or request.query_params.get("source") == "fssp_enforcement_proceedings" + ): + raise + number = int(request.query_params.get(self.page_query_param, "1")) + paginator = self.django_paginator_class( + queryset, self.get_page_size(request) + ) + if number <= paginator.num_pages: + raise + self.page = Page([], number, paginator) + self.request = request + return [] + def get_paginated_response(self, data): response = super().get_paginated_response(data) if not self.page.paginator.count: response.data["meta"]["pagination"]["total_pages"] = 0 + response.data["meta"]["pagination"]["has_previous"] = False return response @@ -286,11 +313,18 @@ def _validate_dates(values, error): def _validate_amounts(values, error): + signed = ( + values.get("source_group") == "enforcement_proceedings" + or values.get("source") == "fssp_enforcement_proceedings" + ) + pattern = ( + r"-?[0-9]{1,18}(?:\.[0-9]{1,2})?" if signed else r"[0-9]{1,18}(?:\.[0-9]{1,2})?" + ) for name in ("amount_from", "amount_to"): if name in values: try: raw = values[name] - if not re.fullmatch(r"[0-9]{1,18}(?:\.[0-9]{1,2})?", raw): + if not re.fullmatch(pattern, raw): raise InvalidOperation values[name] = Decimal(raw) except InvalidOperation: @@ -370,6 +404,22 @@ def source_specific_search(term): """Search selected business fields, never arbitrary raw payload or URLs.""" query = Q() for group, fields in ( + ( + "enforcement_proceedings", + tuple( + f"payload__{field}" + for field in ( + "debtor_raw", + "proceeding_raw", + "document_raw", + "termination_raw", + "service_text", + "subject_raw", + "department_raw", + "bailiff_raw", + ) + ), + ), ( "budget_process_registry", ( diff --git a/src/organizations/test_companies.py b/src/organizations/test_companies.py index fb1b615..0723f15 100644 --- a/src/organizations/test_companies.py +++ b/src/organizations/test_companies.py @@ -44,6 +44,8 @@ CANONICAL_ONLY_TEST_SOURCES = { ParserLoadLog.Source.SRO_MEMBERSHIP_CHECK, } FILE_UPLOAD_ONLY_TEST_SOURCES = { + # Real FSSP snapshots must not be fabricated by the demo command. + ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS, ParserLoadLog.Source.ROPK_SANCTIONS, ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS, ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, diff --git a/tests/apps/organizations/test_canonical_openapi.py b/tests/apps/organizations/test_canonical_openapi.py index 4164b1e..4de7eeb 100644 --- a/tests/apps/organizations/test_canonical_openapi.py +++ b/tests/apps/organizations/test_canonical_openapi.py @@ -36,7 +36,7 @@ def test_discriminated_records_keep_named_ordering_and_old_operations(contract): schemas = contract["components"]["schemas"] for name in ("OrganizationSourceRecord", "OrganizationSourceRecordList"): record = schemas[name] - assert len(record["oneOf"]) == 6 + assert len(record["oneOf"]) == 7 assert record["discriminator"]["propertyName"] == "source_group" for group, record_type in ( ("budget_process_registry", "budget_registry_organization"), @@ -175,3 +175,21 @@ def test_invalid_or_absent_detail_is_typed_error(registry_query, uid, expected): "invalid_uid", "source_record_not_found", } + + +def test_fssp_variant_preserves_signed_nullable_amounts(contract): + schemas = contract["components"]["schemas"] + for kind in ("List", "Detail"): + name = schemas[ + "OrganizationSourceRecord" + if kind == "Detail" + else "OrganizationSourceRecordList" + ]["discriminator"]["mapping"]["enforcement_proceedings"].rsplit("/", 1)[1] + variant = schemas[name] + assert variant["properties"]["amount"]["nullable"] is True + assert variant["properties"]["amount"]["pattern"] == r"^-?[0-9]+\.[0-9]{2}$" + assert variant["properties"]["status"]["enum"] == [ + "unknown", + "ended", + "terminated", + ] diff --git a/tests/apps/organizations/test_fssp_contract.py b/tests/apps/organizations/test_fssp_contract.py new file mode 100644 index 0000000..df88f20 --- /dev/null +++ b/tests/apps/organizations/test_fssp_contract.py @@ -0,0 +1,190 @@ +"""Public FSSP records match the read-only frontend contract.""" + +from datetime import date +from io import BytesIO +from tempfile import TemporaryDirectory +from uuid import UUID + +from apps.parsers.fssp_csv import ENDED_DATASET, FSSP_SOURCE, GENERAL_DATASET +from apps.parsers.fssp_snapshot import FsspInput, import_fssp_snapshots +from apps.parsers.models import ParserLoadLog +from apps.parsers.snapshot_metadata import get_snapshot_artifact +from django.test import TestCase, override_settings +from django.utils import timezone +from organizations.models import Organization +from organizations.source_record_export import _source_group_queryset +from rest_framework.test import APIClient + +from tests.apps.parsers.test_fssp_snapshot import csv_bytes +from tests.apps.user.factories import UserFactory + + +class FsspContractTests(TestCase): + def setUp(self): + temporary = TemporaryDirectory() + self.addCleanup(temporary.cleanup) + settings = override_settings( + MEDIA_ROOT=temporary.name + "/media", + PARSER_PRIVATE_ARTIFACT_ROOT=temporary.name + "/private", + ) + settings.enable() + self.addCleanup(settings.disable) + self.org = Organization.objects.create( + name="АО Проверка", + inn="7707083893", + ogrn="1027700132195", + okpo="12345678", + directory_imported_at=timezone.now(), + ) + self.artifact, _ = import_fssp_snapshots( + inputs=[ + FsspInput( + GENERAL_DATASET, + BytesIO(csv_bytes("general.csv")), + "general.csv", + date(2026, 10, 5), + ), + FsspInput( + ENDED_DATASET, + BytesIO( + csv_bytes( + "ended.csv", + **{ + "The number of enforcement proceeding": "555/26/77001-ИП" + }, + ) + ), + "ended.csv", + date(2026, 10, 5), + ), + ], + load_batch=1, + ) + self.client = APIClient() + self.client.force_authenticate(UserFactory.create_user()) + self.url = "/api/v2/organization-source-records/" + + def test_list_detail_export_same_uid_and_signed_amount(self): + response = self.client.get( + self.url, + { + "source_group": "enforcement_proceedings", + "ordering": "amount", + }, + ) + assert response.status_code == 200, response.data + assert len(response.data["data"]) == 2 + row = response.data["data"][0] + assert row["amount"] == "-12.50" + assert row["payload"]["debt_amount"] == "-12.50" + assert row["payload"]["claim_amount"] == "250.00" + assert row["payload"]["fetched_at"].endswith("Z") + assert row["record_date"] == row["payload"]["initiated_on"] == "2026-10-01" + assert row["title"] == "Исполнительное производство 123/26/77001-ИП" + detail = self.client.get(self.url + row["uid"] + "/") + assert detail.status_code == 200, detail.data + assert detail.data["payload"] == row["payload"] + assert detail.data["organization"]["uid"] == str(self.org.uid) + assert set( + _source_group_queryset( + "enforcement_proceedings", export_year=2026 + ).values_list("uid", flat=True) + ) == {UUID(item["uid"]) for item in response.data["data"]} + + def test_document_raw_search_and_terminated_ordering(self): + response = self.client.get( + self.url, + { + "source_group": "enforcement_proceedings", + "search": "Взыскание сбора", + "ordering": "-payload__terminated_on,payload__document_raw,uid", + }, + ) + assert response.status_code == 200, response.data + assert len(response.data["data"]) == 2 + assert response.data["data"][0]["status"] == "ended" + assert response.data["data"][1]["payload"]["terminated_on"] is None + + def test_snapshot_history_uses_aggregate_not_latest_input(self): + log = ParserLoadLog.objects.create(source=FSSP_SOURCE, batch_id=1) + assert get_snapshot_artifact(log).uid == self.artifact.uid + + def test_signed_amount_filter_is_scoped_to_fssp(self): + response = self.client.get( + self.url, + { + "source": FSSP_SOURCE, + "amount_from": "-20.00", + "amount_to": "0.00", + }, + ) + assert response.status_code == 200, response.data + assert len(response.data["data"]) == 1 + assert response.data["data"][0]["amount"] == "-12.50" + other = self.client.get( + self.url, + { + "source_group": "government_support", + "amount_from": "-20.00", + }, + ) + assert other.status_code == 400 + + def test_payload_has_every_frontend_field_with_nullable_values(self): + from organizations.source_groups import get_source_group_descriptor + + descriptor = get_source_group_descriptor(FSSP_SOURCE) + assert (descriptor.source_group, descriptor.record_type) == ( + "enforcement_proceedings", + "enforcement_proceeding", + ) + response = self.client.get(self.url, {"source": FSSP_SOURCE}) + required = set( + "origin_kind queried_inn debtor_raw debtor_name debtor_inn debtor_legal_address debtor_actual_address proceeding_raw proceeding_number consolidated_proceeding_number initiated_on document_raw document_type document_date document_number document_issuer document_subject_raw creditor_inn termination_raw terminated_on termination_reason termination_article termination_part termination_point service_text service_links subject_raw subject debt_amount claim_amount debt_currency department_raw department_name department_address bailiff_raw bailiff_name bailiff_phones source_url fetched_at parse_warnings".split() + ) + for row in response.data["data"]: + assert required <= row["payload"].keys() + assert row["payload"]["queried_inn"] is None + assert row["payload"]["service_links"] == [] + assert row["payload"]["bailiff_phones"] == [] + + def test_documented_filters_orderings_and_invalid_values(self): + base = { + "source_group": "enforcement_proceedings", + "source": FSSP_SOURCE, + "record_type": "enforcement_proceeding", + "organization": str(self.org.uid), + } + for field in ( + "extension__organization__name", + "record_date", + "amount", + "payload__document_raw", + "payload__terminated_on", + "payload__service_text", + "uid", + ): + for prefix in ("", "-"): + response = self.client.get( + self.url, {**base, "ordering": prefix + field} + ) + assert response.status_code == 200, (field, response.data) + assert len(response.data["data"]) == 2 + response = self.client.get( + self.url, {**base, "date_from": "2026-10-01", "date_to": "2026-10-01"} + ) + assert len(response.data["data"]) == 2 + for params in ( + {"date_from": "2026-02-31"}, + {"organization": "bad"}, + {"date_from": "2026-10-05", "date_to": "2026-10-01"}, + {"page_size": "101"}, + {"ordering": "foreign"}, + ): + assert self.client.get(self.url, {**base, **params}).status_code == 400 + empty = self.client.get(self.url, {**base, "date_from": "2027-01-01"}) + assert empty.data["data"] == [] + assert empty.data["meta"]["pagination"]["total_pages"] == 0 + beyond = self.client.get(self.url, {**base, "page": 99}) + assert beyond.status_code == 200, beyond.data + assert beyond.data["data"] == [] diff --git a/tests/apps/organizations/test_openapi_runtime_fixtures.py b/tests/apps/organizations/test_openapi_runtime_fixtures.py index e63de0a..bb25ba3 100644 --- a/tests/apps/organizations/test_openapi_runtime_fixtures.py +++ b/tests/apps/organizations/test_openapi_runtime_fixtures.py @@ -2,11 +2,13 @@ import json import os +from io import StringIO from pathlib import Path from uuid import uuid4 import pytest from apps.parsers.budget_registry import normalize_budget_record +from apps.parsers.fssp_csv import FSSP_SOURCE, GENERAL_DATASET, iter_fssp_records from apps.parsers.models import ParserSourceArtifact from apps.parsers.sme_support import support_payload from apps.parsers.sro_membership import normalize_sro_membership @@ -108,7 +110,31 @@ def _canonical_records(): lookup_value=organization.ogrn, ), ) - return [budget, sme, sro], raw_budget + fssp_org = Organization.objects.create( + name="АО Проверка", + inn="7707083893", + ogrn="1027700132195", + okpo="12345678", + directory_imported_at=timezone.now(), + opk_registry_membership=False, + ) + csv_fixture = Path(__file__).parents[2] / "fixtures/fssp/general.csv" + row = next( + iter_fssp_records(StringIO(csv_fixture.read_text()), dataset=GENERAL_DATASET) + ) + fssp = _create_record( + fssp_org, + FSSP_SOURCE, + { + "title": row.title, + "record_date": row.record_date, + "amount": row.amount, + "status": row.status, + "url": row.url, + "payload": row.payload, + }, + ) + return [budget, sme, sro, fssp], raw_budget @pytest.mark.django_db @@ -129,8 +155,8 @@ def test_source_record_runtime_fixtures(): return body mixed = capture("mixed-list", "list", client.get(list_url)) - assert len(mixed["data"]) == 3 - assert mixed["meta"]["pagination"]["total_count"] == 3 + assert len(mixed["data"]) == 4 + assert mixed["meta"]["pagination"]["total_count"] == 4 details = {} for record in records: listing = capture( diff --git a/tests/apps/organizations/test_source_export_snapshot_contract.py b/tests/apps/organizations/test_source_export_snapshot_contract.py index f971285..33278e6 100644 --- a/tests/apps/organizations/test_source_export_snapshot_contract.py +++ b/tests/apps/organizations/test_source_export_snapshot_contract.py @@ -66,16 +66,23 @@ def issue(client, group="budget_process_registry", file_format="json"): ) +@pytest.mark.parametrize( + "source,group", + [ + ("budget_ubpandnubp", "budget_process_registry"), + ("fssp_enforcement_proceedings", "enforcement_proceedings"), + ], +) def test_ticket_pins_generation_across_publication_and_rapid_rebuilds( - export_client, tmp_path + export_client, tmp_path, source, group ): - first_artifact = publication() + first_artifact = publication(source=source) first = exports.build_source_record_export_artifacts() - assert first.source_snapshots == {"budget_ubpandnubp": str(first_artifact.pk)} - ticket = issue(export_client) + assert first.source_snapshots == {source: str(first_artifact.pk)} + ticket = issue(export_client, group=group) assert ticket.status_code == 201 - publication(batch=2) - stale = issue(export_client) + publication(source=source, batch=2) + stale = issue(export_client, group=group) assert stale.status_code == 503 assert stale.data["code"] == "source_export_not_ready" exports.build_source_record_export_artifacts() @@ -87,7 +94,7 @@ def test_ticket_pins_generation_across_publication_and_rapid_rebuilds( assert package.generation_id == first.generation_id with zipfile.ZipFile(BytesIO(b"".join(package.archive_chunks))) as archive: assert archive.testzip() is None - assert issue(export_client).status_code == 201 + assert issue(export_client, group=group).status_code == 201 retired = tmp_path / "generations" / first.generation_id / ".retired" old = time.time() - 301 os.utime(retired, (old, old)) @@ -210,7 +217,7 @@ def test_sro_export_keeps_all_years_and_identifier_strings(export_client, file_f record_date="2001-01-01", ) generation = exports.build_source_record_export_artifacts() - assert generation.artifacts_count == 49 + assert generation.artifacts_count == 52 response = issue(export_client, "sro_membership", file_format) assert response.status_code == 201 package = exports.consume_source_record_export_download_ticket( diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index 10ba3cf..b6f9031 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -128,9 +128,9 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): call_command("build_source_record_exports", stdout=command_output) generation = load_current_source_record_export_generation() - self.assertEqual(generation.artifacts_count, 49) + self.assertEqual(generation.artifacts_count, 52) self.assertEqual(generation.export_year, timezone.localdate().year) - self.assertIn('"artifacts_count": 49', command_output.getvalue()) + self.assertIn('"artifacts_count": 52', command_output.getvalue()) def test_sanctions_xlsx_export_uses_source_contract_columns_and_boolean_flags(self): organization = Organization.objects.create( @@ -641,7 +641,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): self.assertEqual( response["X-Source-Export-Generated-At"], generation.generated_at ) - self.assertEqual(generation.artifacts_count, 49) + self.assertEqual(generation.artifacts_count, 52) self.assertIn( 'filename="planned-inspections__financial-indicators_', response["Content-Disposition"], @@ -840,8 +840,8 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): key=lambda item: item.file_name, ) - self.assertEqual(generation.artifacts_count, 49) - self.assertEqual(generation.files_count, 50) + self.assertEqual(generation.artifacts_count, 52) + self.assertEqual(generation.files_count, 53) self.assertEqual( [item.file_name for item in artifacts], [ @@ -1065,7 +1065,7 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): first_generation = build_source_record_export_artifacts() current_generation = load_current_source_record_export_generation() - self.assertEqual(first_generation.artifacts_count, 49) + self.assertEqual(first_generation.artifacts_count, 52) self.assertEqual( current_generation.generation_id, first_generation.generation_id ) diff --git a/tests/apps/organizations/test_tasks.py b/tests/apps/organizations/test_tasks.py index 58e0545..acca47e 100644 --- a/tests/apps/organizations/test_tasks.py +++ b/tests/apps/organizations/test_tasks.py @@ -147,7 +147,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase): result = refresh_source_record_export_artifacts() self.assertEqual(result["status"], "success") - self.assertEqual(result["artifacts_count"], 49) + self.assertEqual(result["artifacts_count"], 52) self.assertEqual(result["export_year"], timezone.localdate().year) self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY)) diff --git a/tests/apps/organizations/test_test_companies_commands.py b/tests/apps/organizations/test_test_companies_commands.py index d605f5d..1d4e913 100644 --- a/tests/apps/organizations/test_test_companies_commands.py +++ b/tests/apps/organizations/test_test_companies_commands.py @@ -79,6 +79,7 @@ class TestCompaniesCommandsTest(TestCase): ParserLoadLog.Source.ROPK_SANCTIONS, ParserLoadLog.Source.INTEGRATED_STRUCTURE_DOCUMENTS, ParserLoadLog.Source.FEDRESURS_COMPANY_PROFILE, + ParserLoadLog.Source.FSSP_ENFORCEMENT_PROCEEDINGS, } } for company in companies: @@ -273,7 +274,7 @@ class TestCompaniesCommandsTest(TestCase): ) self.assertEqual( OrganizationSourceRecord.objects.count(), - 20 * (len(ParserLoadLog.Source) - 3), + 20 * (len(ParserLoadLog.Source) - 4), ) def test_create_replaces_stale_source_record_for_same_legacy_row(self): diff --git a/tests/apps/parsers/test_fssp_csv.py b/tests/apps/parsers/test_fssp_csv.py index ef0e8d0..0ee7f6f 100644 --- a/tests/apps/parsers/test_fssp_csv.py +++ b/tests/apps/parsers/test_fssp_csv.py @@ -132,3 +132,23 @@ def test_iterator_does_not_read_next_row_until_requested(): assert next(iterator).inn == "7707083893" with pytest.raises(SnapshotValidationError): next(iterator) + + +def test_real_ended_iso_date_and_publisher_reason_labels(): + row = next( + records( + fixture_text( + "ended.csv", + **{ + "Date, complete IP reason": "2025-10-25, ст. 46 п. 1 п.п. 4", + }, + ), + dataset=ENDED, + ) + ) + assert row.payload["terminated_on"] == "2025-10-25" + assert row.payload["termination_article"] == "46" + assert row.payload["termination_part"] is None + assert row.payload["termination_point"] is None + assert "unparsed_termination" in row.payload["parse_warnings"] + assert row.payload["termination_raw"] == "2025-10-25, ст. 46 п. 1 п.п. 4" diff --git a/tests/apps/parsers/test_fssp_refresh.py b/tests/apps/parsers/test_fssp_refresh.py new file mode 100644 index 0000000..c5b7ba5 --- /dev/null +++ b/tests/apps/parsers/test_fssp_refresh.py @@ -0,0 +1,73 @@ +"""Two entrypoints, one FSSP task, no caller-supplied download location.""" + +from importlib import import_module +from unittest.mock import patch + +import pytest +from apps.core.models import BackgroundJob +from apps.parsers.source_cards import SourceCardService +from core.celery import app as celery_app +from rest_framework.test import APIClient + +from tests.apps.user.factories import UserFactory + + +@pytest.mark.django_db +def test_refresh_and_run_share_one_job_and_reject_second_run(): + import_module("apps.parsers.tasks") + assert "parsers.fssp_enforcement_proceedings.refresh" in celery_app.tasks + task = celery_app.tasks["parsers.fssp_enforcement_proceedings.refresh"] + client = APIClient() + client.force_authenticate(UserFactory.create_user(is_staff=True)) + with patch.object(task, "apply_async"): + first = client.post( + "/api/v1/sources/fssp-enforcement-proceedings/refresh/", {}, format="json" + ) + assert first.status_code == 202, first.data + second = client.post( + "/api/v1/parsers/run/fssp_enforcement_proceedings/", {}, format="json" + ) + assert second.status_code == 409, second.data + assert BackgroundJob.objects.count() == 1 + assert SourceCardService.get_parser_sources_by_card_slug( + "fssp-enforcement-proceedings" + ) == ["fssp_enforcement_proceedings"] + + +@pytest.mark.django_db +def test_refresh_permissions_and_no_download_location(): + client = APIClient() + client.force_authenticate(UserFactory.create_user()) + urls = ( + "/api/v1/sources/fssp-enforcement-proceedings/refresh/", + "/api/v1/parsers/run/fssp_enforcement_proceedings/", + ) + for url in urls: + assert client.post(url, {}, format="json").status_code == 403 + client.force_authenticate(UserFactory.create_user(is_staff=True)) + for url in urls: + response = client.post( + url, {"params": {"file_url": "https://foreign.invalid"}}, format="json" + ) + assert response.status_code == 400, response.data + assert BackgroundJob.objects.count() == 0 + + +@pytest.mark.django_db +def test_fssp_unknown_total_progress_does_not_report_completion(): + from apps.core.models import JobStatus + from apps.parsers.tasks_registry_snapshots import _update_snapshot_progress + + job = BackgroundJob.objects.create( + task_id="fssp-progress", + task_name="parsers.fssp_enforcement_proceedings.refresh", + status=JobStatus.STARTED, + meta={"source": "fssp_enforcement_proceedings"}, + ) + _update_snapshot_progress(job, 10000, 0, 12, 3) + job.refresh_from_db() + assert job.progress == 5 + assert job.meta["raw_records_count"] == 10000 + assert job.meta["matched_records_count"] == 12 + assert job.meta["quarantined_records_count"] == 3 + assert "организаций" not in job.progress_message diff --git a/tests/apps/parsers/test_fssp_snapshot.py b/tests/apps/parsers/test_fssp_snapshot.py index a7cab29..8538802 100644 --- a/tests/apps/parsers/test_fssp_snapshot.py +++ b/tests/apps/parsers/test_fssp_snapshot.py @@ -79,6 +79,15 @@ class FsspSnapshotTests(TestCase): ) ) + def test_invalid_canonical_requisites_are_quarantined(self): + self.org.ogrn = "bad" + self.org.okpo = "1234567890" + self.org.save() + artifact, result = self.run_import() + assert result.published == 0 + assert result.quarantined == 2 + assert artifact.metadata["reasons"]["invalid_canonical_requisites"] == 2 + def test_two_inputs_manifest_and_stable_uid(self): artifact, result = self.run_import() assert result.parsed == 2 diff --git a/tests/apps/parsers/test_refresh_concurrency.py b/tests/apps/parsers/test_refresh_concurrency.py index c56e790..7826c5d 100644 --- a/tests/apps/parsers/test_refresh_concurrency.py +++ b/tests/apps/parsers/test_refresh_concurrency.py @@ -34,6 +34,7 @@ def _parallel(actions): ("sme-support-recipients-registry", "fns_sme_support_recipients"), ("budget-process-registry", "budget_ubpandnubp"), ("sro-membership-check", "sro_membership_check"), + ("fssp-enforcement-proceedings", "fssp_enforcement_proceedings"), ], ) def test_two_simultaneous_refreshes_dispatch_exactly_one_task(slug, source): diff --git a/tests/apps/parsers/test_snapshot_public_contract.py b/tests/apps/parsers/test_snapshot_public_contract.py index e3cd5a1..8da759b 100644 --- a/tests/apps/parsers/test_snapshot_public_contract.py +++ b/tests/apps/parsers/test_snapshot_public_contract.py @@ -24,6 +24,12 @@ from rest_framework.test import APIClient from tests.apps.user.factories import UserFactory SOURCES = ( + ( + "fssp-enforcement-proceedings", + "fssp_enforcement_proceedings", + "enforcement_proceedings", + "enforcement_proceeding", + ), ( "budget-process-registry", "budget_ubpandnubp", diff --git a/tests/apps/parsers/test_source_cards_service.py b/tests/apps/parsers/test_source_cards_service.py index 0d2f3ef..586155a 100644 --- a/tests/apps/parsers/test_source_cards_service.py +++ b/tests/apps/parsers/test_source_cards_service.py @@ -97,6 +97,7 @@ class SourceCardServiceUnitTest(TestCase): "sro-membership-check", "integrated-structure-documents", "fedresurs-company-profile", + "fssp-enforcement-proceedings", ], ) self.assertEqual( @@ -119,6 +120,7 @@ class SourceCardServiceUnitTest(TestCase): "Проверка членства в СРО", "Документы интегрированных структур", "Федресурс: сведения о юридических лицах", + "ФССП: Банк данных исполнительных производств", ], ) diff --git a/tools/openapi/build.mjs b/tools/openapi/build.mjs index d409597..0d28c0e 100644 --- a/tools/openapi/build.mjs +++ b/tools/openapi/build.mjs @@ -6,7 +6,7 @@ import converter from 'swagger2openapi'; import validator from 'oas-validator'; const base = fileURLToPath(new URL('../..', import.meta.url)); -const input = process.argv[2] || resolve(base, 'tools/openapi/swagger.json'); +const input = process.argv.slice(2).find(argument => !argument.startsWith('--')) || resolve(base, 'tools/openapi/swagger.json'); const output = resolve(base, 'src/core/openapi.json'); const check = process.argv.includes('--check'); const swagger = JSON.parse(readFileSync(input, 'utf8')); @@ -26,6 +26,7 @@ const variants = [ ['SroMembership', 'sro_membership', 'sro_membership_check', 'sro_membership', 'SroMembership'], ]; const extendedVariants = [ + ['FsspEnforcementProceeding', 'enforcement_proceedings', 'fssp_enforcement_proceedings', 'enforcement_proceeding', 'FsspEnforcementProceeding'], ['IntegratedStructure', 'integrated_structure_documents', 'integrated_structures', 'integrated_structure', 'IntegratedStructure'], ['FedresursCompanyProfile', 'fedresurs_company_profile', 'fedresurs_company_profile', 'company_profile', 'FedresursCompanyProfile'], ]; @@ -49,6 +50,15 @@ for (const field of schemas.SroSourceRecordOrganization.required) { delete schemas.SroSourceRecordOrganization.properties[field].nullable; } +schemas.FsspSourceRecordOrganization = clone(schemas.PublishedSourceRecordOrganization); +for (const field of ['name', 'inn', 'ogrn', 'okpo']) { + delete schemas.FsspSourceRecordOrganization.properties[field].nullable; + schemas.FsspSourceRecordOrganization.properties[field].minLength = 1; +} +for (const [field, length] of [['inn', 10], ['ogrn', 13], ['okpo', 8]]) { + schemas.FsspSourceRecordOrganization.properties[field].pattern = `^[0-9]{${length}}$`; +} + for (const [schemaName, kind] of [ ['OrganizationSourceRecordList', 'List'], ['OrganizationSourceRecord', 'Detail'], @@ -134,6 +144,15 @@ for (const [prefix, group, source, recordType, payloadPrefix] of extendedVariant schema.properties.status = { type: 'string' }; schema.properties.payload = ref(payloadName); schema.properties.organization = ref('SroSourceRecordOrganization'); + if (group === 'enforcement_proceedings') { + schema.required = [...new Set([...schema.required, 'extension_uid', 'record_date', 'amount', 'url', 'legacy_model', 'legacy_pk', 'load_batch', 'financial_lines'])]; + schema.properties.status = { type: 'string', enum: ['unknown', 'ended', 'terminated'] }; + schema.properties.amount = { type: 'string', pattern: '^-?[0-9]+\\.[0-9]{2}$', nullable: true }; + schema.properties.record_date = { type: 'string', format: 'date', nullable: true }; + for (const field of ['legacy_model', 'legacy_pk']) schema.properties[field] = { type: 'string', enum: [''] }; + schema.properties.financial_lines = { type: 'array', items: {}, maxItems: 0 }; + schema.properties.organization = ref('FsspSourceRecordOrganization'); + } schemas[variantName] = schema; schemas[schemaName].oneOf.push(ref(variantName)); schemas[schemaName].discriminator.mapping[group] = ref(variantName).$ref; diff --git a/tools/openapi/check-runtime.mjs b/tools/openapi/check-runtime.mjs index 75c5cde..b5a5e6e 100644 --- a/tools/openapi/check-runtime.mjs +++ b/tools/openapi/check-runtime.mjs @@ -17,7 +17,7 @@ const schemas = new Map([ ]); const requiredCases = new Set([ 'mixed-list', 'empty-list', 'invalid-list', 'invalid-detail', 'missing-detail', - ...['budget_ubpandnubp', 'fns_sme_support_recipients', 'sro_membership_check'] + ...['budget_ubpandnubp', 'fns_sme_support_recipients', 'sro_membership_check', 'fssp_enforcement_proceedings'] .flatMap(source => [`${source}-list`, `${source}-detail`]), ]); diff --git a/tools/openapi/source-record-extensions.json b/tools/openapi/source-record-extensions.json index ab843ba..34331ff 100644 --- a/tools/openapi/source-record-extensions.json +++ b/tools/openapi/source-record-extensions.json @@ -461,5 +461,515 @@ "message_details", "lineage" ] + }, + "FsspEnforcementProceedingListPayload": { + "type": "object", + "properties": { + "queried_inn": { + "type": "string", + "nullable": true + }, + "debtor_name": { + "type": "string", + "nullable": true + }, + "debtor_legal_address": { + "type": "string", + "nullable": true + }, + "debtor_actual_address": { + "type": "string", + "nullable": true + }, + "consolidated_proceeding_number": { + "type": "string", + "nullable": true + }, + "document_raw": { + "type": "string", + "nullable": true + }, + "document_type": { + "type": "string", + "nullable": true + }, + "document_number": { + "type": "string", + "nullable": true + }, + "document_issuer": { + "type": "string", + "nullable": true + }, + "document_subject_raw": { + "type": "string", + "nullable": true + }, + "creditor_inn": { + "type": "string", + "nullable": true + }, + "termination_raw": { + "type": "string", + "nullable": true + }, + "termination_reason": { + "type": "string", + "nullable": true + }, + "termination_article": { + "type": "string", + "nullable": true + }, + "termination_part": { + "type": "string", + "nullable": true + }, + "termination_point": { + "type": "string", + "nullable": true + }, + "service_text": { + "type": "string", + "nullable": true + }, + "subject_raw": { + "type": "string", + "nullable": true + }, + "subject": { + "type": "string", + "nullable": true + }, + "debt_amount": { + "type": "string", + "nullable": true, + "pattern": "^-?[0-9]+\\.[0-9]{2}$" + }, + "claim_amount": { + "type": "string", + "nullable": true, + "pattern": "^[0-9]+\\.[0-9]{2}$" + }, + "debt_currency": { + "type": "string", + "nullable": true + }, + "department_raw": { + "type": "string", + "nullable": true + }, + "department_name": { + "type": "string", + "nullable": true + }, + "department_address": { + "type": "string", + "nullable": true + }, + "bailiff_raw": { + "type": "string", + "nullable": true + }, + "bailiff_name": { + "type": "string", + "nullable": true + }, + "debtor_raw": { + "type": "string" + }, + "debtor_inn": { + "type": "string" + }, + "proceeding_raw": { + "type": "string" + }, + "proceeding_number": { + "type": "string" + }, + "source_url": { + "type": "string" + }, + "dataset": { + "type": "string" + }, + "initiated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "document_date": { + "type": "string", + "format": "date", + "nullable": true + }, + "terminated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "origin_kind": { + "type": "string", + "enum": [ + "web_search", + "open_data_csv" + ], + "nullable": true + }, + "fetched_at": { + "type": "string", + "format": "date-time" + }, + "service_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "label", + "url" + ] + } + }, + "bailiff_phones": { + "type": "array", + "items": { + "type": "string" + } + }, + "parse_warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "raw_columns": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "source_row_number": { + "type": "integer" + }, + "provenance": { + "type": "object", + "additionalProperties": true + }, + "observations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + } + }, + "required": [ + "queried_inn", + "debtor_name", + "debtor_legal_address", + "debtor_actual_address", + "consolidated_proceeding_number", + "document_raw", + "document_type", + "document_number", + "document_issuer", + "document_subject_raw", + "creditor_inn", + "termination_raw", + "termination_reason", + "termination_article", + "termination_part", + "termination_point", + "service_text", + "subject_raw", + "subject", + "debt_amount", + "claim_amount", + "debt_currency", + "department_raw", + "department_name", + "department_address", + "bailiff_raw", + "bailiff_name", + "debtor_raw", + "debtor_inn", + "proceeding_raw", + "proceeding_number", + "source_url", + "dataset", + "initiated_on", + "document_date", + "terminated_on", + "origin_kind", + "fetched_at", + "service_links", + "bailiff_phones", + "parse_warnings", + "raw_columns", + "source_row_number" + ] + }, + "FsspEnforcementProceedingDetailPayload": { + "type": "object", + "properties": { + "queried_inn": { + "type": "string", + "nullable": true + }, + "debtor_name": { + "type": "string", + "nullable": true + }, + "debtor_legal_address": { + "type": "string", + "nullable": true + }, + "debtor_actual_address": { + "type": "string", + "nullable": true + }, + "consolidated_proceeding_number": { + "type": "string", + "nullable": true + }, + "document_raw": { + "type": "string", + "nullable": true + }, + "document_type": { + "type": "string", + "nullable": true + }, + "document_number": { + "type": "string", + "nullable": true + }, + "document_issuer": { + "type": "string", + "nullable": true + }, + "document_subject_raw": { + "type": "string", + "nullable": true + }, + "creditor_inn": { + "type": "string", + "nullable": true + }, + "termination_raw": { + "type": "string", + "nullable": true + }, + "termination_reason": { + "type": "string", + "nullable": true + }, + "termination_article": { + "type": "string", + "nullable": true + }, + "termination_part": { + "type": "string", + "nullable": true + }, + "termination_point": { + "type": "string", + "nullable": true + }, + "service_text": { + "type": "string", + "nullable": true + }, + "subject_raw": { + "type": "string", + "nullable": true + }, + "subject": { + "type": "string", + "nullable": true + }, + "debt_amount": { + "type": "string", + "nullable": true, + "pattern": "^-?[0-9]+\\.[0-9]{2}$" + }, + "claim_amount": { + "type": "string", + "nullable": true, + "pattern": "^[0-9]+\\.[0-9]{2}$" + }, + "debt_currency": { + "type": "string", + "nullable": true + }, + "department_raw": { + "type": "string", + "nullable": true + }, + "department_name": { + "type": "string", + "nullable": true + }, + "department_address": { + "type": "string", + "nullable": true + }, + "bailiff_raw": { + "type": "string", + "nullable": true + }, + "bailiff_name": { + "type": "string", + "nullable": true + }, + "debtor_raw": { + "type": "string" + }, + "debtor_inn": { + "type": "string" + }, + "proceeding_raw": { + "type": "string" + }, + "proceeding_number": { + "type": "string" + }, + "source_url": { + "type": "string" + }, + "dataset": { + "type": "string" + }, + "initiated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "document_date": { + "type": "string", + "format": "date", + "nullable": true + }, + "terminated_on": { + "type": "string", + "format": "date", + "nullable": true + }, + "origin_kind": { + "type": "string", + "enum": [ + "web_search", + "open_data_csv" + ], + "nullable": true + }, + "fetched_at": { + "type": "string", + "format": "date-time" + }, + "service_links": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "url": { + "type": "string" + } + }, + "required": [ + "label", + "url" + ] + } + }, + "bailiff_phones": { + "type": "array", + "items": { + "type": "string" + } + }, + "parse_warnings": { + "type": "array", + "items": { + "type": "string" + } + }, + "raw_columns": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "source_row_number": { + "type": "integer" + }, + "provenance": { + "type": "object", + "additionalProperties": true + }, + "observations": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true + } + } + }, + "required": [ + "queried_inn", + "debtor_name", + "debtor_legal_address", + "debtor_actual_address", + "consolidated_proceeding_number", + "document_raw", + "document_type", + "document_number", + "document_issuer", + "document_subject_raw", + "creditor_inn", + "termination_raw", + "termination_reason", + "termination_article", + "termination_part", + "termination_point", + "service_text", + "subject_raw", + "subject", + "debt_amount", + "claim_amount", + "debt_currency", + "department_raw", + "department_name", + "department_address", + "bailiff_raw", + "bailiff_name", + "debtor_raw", + "debtor_inn", + "proceeding_raw", + "proceeding_number", + "source_url", + "dataset", + "initiated_on", + "document_date", + "terminated_on", + "origin_kind", + "fetched_at", + "service_links", + "bailiff_phones", + "parse_warnings", + "raw_columns", + "source_row_number" + ] } }