From eb170ff914b8909c3ea1a0fa4df531a3d04b14fc Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchriakov Date: Sun, 9 Aug 2026 12:23:37 +0200 Subject: [PATCH] feat: restore report files and export claim amounts --- .../commands/generate_test_reports.py | 2 + .../commands/restore_report_upload_files.py | 116 ++++++++++++++++++ src/apps/exchange/services.py | 5 + .../0009_arbitration_case_claim_amount.py | 21 ++++ src/apps/external_data/models.py | 7 ++ src/apps/external_data/serializers.py | 1 + .../external_data/source_record_export.py | 2 + ...est_restore_report_upload_files_command.py | 80 ++++++++++++ tests/apps/exchange/test_api.py | 5 + tests/apps/external_data/factories.py | 3 + tests/apps/external_data/test_api.py | 1 + .../test_source_record_export.py | 8 ++ 12 files changed, 251 insertions(+) create mode 100644 src/apps/core/management/commands/restore_report_upload_files.py create mode 100644 src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py create mode 100644 tests/apps/core/test_restore_report_upload_files_command.py diff --git a/src/apps/core/management/commands/generate_test_reports.py b/src/apps/core/management/commands/generate_test_reports.py index 1d928d5..83f316f 100644 --- a/src/apps/core/management/commands/generate_test_reports.py +++ b/src/apps/core/management/commands/generate_test_reports.py @@ -605,6 +605,8 @@ class Command(BaseAppCommand): "court_name": "Арбитражный суд города Москвы", "party_role": "defendant" if index % 2 == 0 else "plaintiff", "status": "hearing_scheduled" if index % 4 else "decision_rendered", + "claim_amount": Decimal("500000.00") + + Decimal(index) * Decimal("25000.00"), "decision_date": date( date.today().year, ((index + 2) % 12) + 1, 27 ), diff --git a/src/apps/core/management/commands/restore_report_upload_files.py b/src/apps/core/management/commands/restore_report_upload_files.py new file mode 100644 index 0000000..70451db --- /dev/null +++ b/src/apps/core/management/commands/restore_report_upload_files.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import hashlib +import json +from collections import defaultdict +from pathlib import Path + +from apps.core.management.commands.base import BaseAppCommand +from apps.core.models import ReportUpload +from django.core.files import File +from django.core.management.base import CommandError + + +def _path_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _storage_sha256(storage, name: str) -> str: + digest = hashlib.sha256() + with storage.open(name, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _build_source_index(source_directories: list[Path]) -> dict[str, list[Path]]: + sources_by_hash: dict[str, list[Path]] = defaultdict(list) + for source_directory in source_directories: + for candidate in sorted(source_directory.rglob("*")): + if candidate.is_file(): + sources_by_hash[_path_sha256(candidate)].append(candidate) + return dict(sources_by_hash) + + +def _restore_upload( + upload: ReportUpload, + *, + sources_by_hash: dict[str, list[Path]], + dry_run: bool, +) -> str: + target_name = str(upload.original_file.name or "").strip() + expected_hash = upload.file_hash.strip().lower() + if not target_name or not expected_hash: + return "unmatched" + + storage = upload.original_file.storage + if storage.exists(target_name): + if _storage_sha256(storage, target_name) == expected_hash: + return "already_present" + return "hash_mismatch" + + candidates = sources_by_hash.get(expected_hash, []) + if not candidates: + return "unmatched" + if dry_run: + return "restored" + + source_path = candidates[0] + with source_path.open("rb") as source_handle: + saved_name = storage.save(target_name, File(source_handle)) + if saved_name != target_name: + storage.delete(saved_name) + raise CommandError( + f"Storage changed target name for report upload {upload.id}." + ) + if _storage_sha256(storage, target_name) != expected_hash: + storage.delete(target_name) + raise CommandError(f"Hash verification failed for report upload {upload.id}.") + return "restored" + + +class Command(BaseAppCommand): + help = ( + "Restore missing original report uploads by matching trusted files by SHA-256." + ) + + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument( + "--source-dir", + action="append", + required=True, + help="Trusted directory to scan recursively; may be supplied multiple times.", + ) + + def execute_command(self, *args, **options) -> str: + source_directories = [Path(value).resolve() for value in options["source_dir"]] + missing_directories = [path for path in source_directories if not path.is_dir()] + if missing_directories: + raise CommandError( + "Source directories do not exist: " + + ", ".join(str(path) for path in missing_directories) + ) + + sources_by_hash = _build_source_index(source_directories) + stats = { + "restored": 0, + "already_present": 0, + "unmatched": 0, + "hash_mismatch": 0, + } + for upload in ReportUpload.objects.order_by("created_at").iterator(): + outcome = _restore_upload( + upload, + sources_by_hash=sources_by_hash, + dry_run=self.dry_run, + ) + stats[outcome] += 1 + + result = json.dumps(stats, ensure_ascii=False, sort_keys=True) + self.log_info(result) + return result diff --git a/src/apps/exchange/services.py b/src/apps/exchange/services.py index 50bcfe9..d577596 100644 --- a/src/apps/exchange/services.py +++ b/src/apps/exchange/services.py @@ -1308,6 +1308,11 @@ class ExchangePackageImportService: "court_name": cls._clean_string(row.get("court_name")), "party_role": cls._clean_string(row.get("party_role")), "status": cls._clean_string(row.get("status")), + "claim_amount": cls._parse_decimal_value( + row.get("claim_amount"), + field_name="claim_amount", + allow_null=True, + ), "decision_date": cls._parse_date_value( row.get("decision_date"), field_name="decision_date", diff --git a/src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py b/src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py new file mode 100644 index 0000000..ffd95ec --- /dev/null +++ b/src/apps/external_data/migrations/0009_arbitration_case_claim_amount.py @@ -0,0 +1,21 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("external_data", "0008_export_date_indexes"), + ] + + operations = [ + migrations.AddField( + model_name="arbitrationcase", + name="claim_amount", + field=models.DecimalField( + blank=True, + decimal_places=2, + max_digits=20, + null=True, + verbose_name="размер иска", + ), + ), + ] diff --git a/src/apps/external_data/models.py b/src/apps/external_data/models.py index ede314e..bb93c8b 100644 --- a/src/apps/external_data/models.py +++ b/src/apps/external_data/models.py @@ -159,6 +159,13 @@ class ArbitrationCase(UUIDPrimaryKeyMixin, TimestampMixin, models.Model): court_name = models.CharField(_("суд"), max_length=255) party_role = models.CharField(_("роль стороны"), max_length=64, db_index=True) status = models.CharField(_("статус"), max_length=64, db_index=True) + claim_amount = models.DecimalField( + _("размер иска"), + max_digits=20, + decimal_places=2, + null=True, + blank=True, + ) decision_date = models.DateField(_("дата решения"), db_index=True) class Meta: diff --git a/src/apps/external_data/serializers.py b/src/apps/external_data/serializers.py index 26364d3..4711261 100644 --- a/src/apps/external_data/serializers.py +++ b/src/apps/external_data/serializers.py @@ -113,6 +113,7 @@ class ArbitrationCaseSerializer(serializers.ModelSerializer): "court_name", "party_role", "status", + "claim_amount", "decision_date", ] diff --git a/src/apps/external_data/source_record_export.py b/src/apps/external_data/source_record_export.py index ca01702..f9adc60 100644 --- a/src/apps/external_data/source_record_export.py +++ b/src/apps/external_data/source_record_export.py @@ -332,12 +332,14 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = { "court_name", "party_role", "status", + "claim_amount", "decision_date", ), source="arbitration", external_id_field="case_number", title_field="case_number", record_date_field="decision_date", + amount_field="claim_amount", status_field="status", title_prefix="Дело ", payload_aliases=(("court_name", "court"), ("party_role", "role")), diff --git a/tests/apps/core/test_restore_report_upload_files_command.py b/tests/apps/core/test_restore_report_upload_files_command.py new file mode 100644 index 0000000..0e7bdd8 --- /dev/null +++ b/tests/apps/core/test_restore_report_upload_files_command.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import hashlib +import tempfile +from pathlib import Path + +from apps.core.models import ReportUpload, ReportUploadStatus +from django.core.management import call_command +from django.test import TestCase, override_settings + + +class RestoreReportUploadFilesCommandTest(TestCase): + def _upload(self, *, content: bytes) -> ReportUpload: + digest = hashlib.sha256(content).hexdigest() + upload = ReportUpload.objects.create( + form="f1", + load_batch=101, + original_file="report_uploads/f1/restored/report.xlsx", + file_name="report.xlsx", + content_type=( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ), + file_size=len(content), + file_hash=digest, + status=ReportUploadStatus.SUCCESS, + ) + return upload + + def test_dry_run_then_restore_and_repeat(self): + content = b"exact original workbook bytes" + upload = self._upload(content=content) + + with tempfile.TemporaryDirectory() as media_dir, tempfile.TemporaryDirectory() as source_dir: + source_path = Path(source_dir) / "original.xlsx" + source_path.write_bytes(content) + with override_settings(MEDIA_ROOT=media_dir): + dry_run_result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + dry_run=True, + ) + self.assertIn('"restored": 1', dry_run_result) + self.assertFalse( + upload.original_file.storage.exists(upload.original_file.name) + ) + + restore_result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + ) + self.assertIn('"restored": 1', restore_result) + with upload.original_file.storage.open( + upload.original_file.name, "rb" + ) as handle: + self.assertEqual(handle.read(), content) + + repeat_result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + ) + self.assertIn('"already_present": 1', repeat_result) + + def test_existing_hash_mismatch_is_not_overwritten(self): + content = b"expected workbook" + upload = self._upload(content=content) + + with tempfile.TemporaryDirectory() as media_dir, tempfile.TemporaryDirectory() as source_dir: + (Path(source_dir) / "original.xlsx").write_bytes(content) + target_path = Path(media_dir) / upload.original_file.name + target_path.parent.mkdir(parents=True) + target_path.write_bytes(b"different bytes") + + with override_settings(MEDIA_ROOT=media_dir): + result = call_command( + "restore_report_upload_files", + source_dir=[source_dir], + ) + + self.assertIn('"hash_mismatch": 1', result) + self.assertEqual(target_path.read_bytes(), b"different bytes") diff --git a/tests/apps/exchange/test_api.py b/tests/apps/exchange/test_api.py index 6a6d7c0..3167e91 100644 --- a/tests/apps/exchange/test_api.py +++ b/tests/apps/exchange/test_api.py @@ -316,6 +316,7 @@ def build_exchange_payload() -> dict[str, list[dict[str, object]]]: "party_role": "ответчик", "status": "in_progress", "decision_date": "2026-03-25", + "claim_amount": "1250000.50", } ], "bankruptcy_procedures": [ @@ -463,6 +464,10 @@ class ExchangePackageApiTest(APITestCase): self.assertEqual(FinancialReport.objects.count(), 1) self.assertEqual(FinancialReportLine.objects.count(), 1) self.assertEqual(ArbitrationCase.objects.count(), 1) + self.assertEqual( + ArbitrationCase.objects.get().claim_amount, + Decimal("1250000.50"), + ) self.assertEqual(BankruptcyProcedure.objects.count(), 1) self.assertEqual( BankruptcyProcedure._meta.get_field("status").max_length, diff --git a/tests/apps/external_data/factories.py b/tests/apps/external_data/factories.py index f6619b1..1089ad8 100644 --- a/tests/apps/external_data/factories.py +++ b/tests/apps/external_data/factories.py @@ -101,6 +101,9 @@ class ArbitrationCaseFactory(factory.django.DjangoModelFactory): court_name = "Арбитражный суд города Москвы" party_role = "defendant" status = "hearing_scheduled" + claim_amount = factory.LazyAttribute( + lambda _: fake.pydecimal(left_digits=8, right_digits=2, positive=True) + ) decision_date = factory.LazyAttribute(lambda _: fake.date_this_year()) diff --git a/tests/apps/external_data/test_api.py b/tests/apps/external_data/test_api.py index 6aa2ec6..4202ccd 100644 --- a/tests/apps/external_data/test_api.py +++ b/tests/apps/external_data/test_api.py @@ -124,6 +124,7 @@ class ExternalDataApiTest(APITestCase): self.assertEqual(procurement_response.data["count"], 1) self.assertEqual(arbitration_response.status_code, status.HTTP_200_OK) self.assertEqual(arbitration_response.data["count"], 1) + self.assertIn("claim_amount", arbitration_response.data["results"][0]) def test_corporation_memberships_filter(self): InformationSecurityRegistryEntryFactory( diff --git a/tests/apps/external_data/test_source_record_export.py b/tests/apps/external_data/test_source_record_export.py index d396acf..c04aa32 100644 --- a/tests/apps/external_data/test_source_record_export.py +++ b/tests/apps/external_data/test_source_record_export.py @@ -164,6 +164,14 @@ class SourceRecordExportApiTest(APITestCase): arbitration_row["record_date"], arbitration_case.decision_date.isoformat() ) self.assertEqual(arbitration_row["status"], arbitration_row["payload.status"]) + self.assertEqual( + arbitration_row["amount"], + str(arbitration_case.claim_amount), + ) + self.assertEqual( + arbitration_row["payload.claim_amount"], + str(arbitration_case.claim_amount), + ) self.assertEqual(arbitration_row["payload.role"], "defendant") self.assertIn("payload.court", arbitration_row) self.assertEqual(arbitration_row["payload.source"], "arbitration")