From a00ecec9364f33f24cffc1009fbb42808f74e105 Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 28 Sep 2026 12:06:09 +0200 Subject: [PATCH] feat: add separate all-years source export task --- docs/all-years-source-exports.md | 15 +++ .../commands/build_source_record_exports.py | 12 +- src/organizations/source_record_export.py | 39 ++++-- src/organizations/tasks.py | 14 +- .../test_source_record_export.py | 120 ++++++++++++++++++ 5 files changed, 188 insertions(+), 12 deletions(-) create mode 100644 docs/all-years-source-exports.md diff --git a/docs/all-years-source-exports.md b/docs/all-years-source-exports.md new file mode 100644 index 0000000..974bb20 --- /dev/null +++ b/docs/all-years-source-exports.md @@ -0,0 +1,15 @@ +# Выгрузка за все годы + +Годовая задача `organizations.tasks.refresh_source_record_export_artifacts` сохраняет прежнее поведение. + +Новая задача `organizations.tasks.refresh_all_years_source_record_export_artifacts` собирает все доступные записи каждого поддерживаемого источника без ограничения по году, включая все финансовые строки. Для источников с заменяемыми snapshot выгружается текущее опубликованное состояние: удалённые прежние snapshot не восстанавливаются. + +Результат размещается в `SOURCE_RECORD_EXPORT_DIRECTORY/all-years`. У него отдельные `current.json`, `generations` и очистка старых поколений. Публикация атомарна; незавершённая сборка не заменяет опубликованные файлы. Общий lock с годовой задачей ограничивает одновременную нагрузку. Manifest содержит `export_scope: all_years` и `export_year: null`; годовой manifest — `export_scope: current_year` и числовой год. Старые годовые manifest без `export_scope` продолжают читаться. + +Ручной запуск: + +```sh +python src/manage.py build_source_record_exports --all-years +``` + +Периодический запуск настраивается отдельной записью django-celery-beat либо системным таймером на контуре без Beat. Задачу следует ставить после годовой сборки. Имена файлов, группировка JSON по ОКПО и форматы CSV/XLSX/JSON сохраняются. API скачивания продолжает отдавать годовой пакет; доступ к полному пакету предоставляется отдельно. diff --git a/src/organizations/management/commands/build_source_record_exports.py b/src/organizations/management/commands/build_source_record_exports.py index 6a7c8dc..ef53751 100644 --- a/src/organizations/management/commands/build_source_record_exports.py +++ b/src/organizations/management/commands/build_source_record_exports.py @@ -15,8 +15,18 @@ class Command(BaseAppCommand): help = "Формирует готовые CSV/XLSX/JSON выгрузки источников" use_transaction = False + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument( + "--all-years", + action="store_true", + help="Все доступные годы в отдельном каталоге all-years", + ) + def execute_command(self, *args, **options) -> str: - generation = build_source_record_export_artifacts() + generation = build_source_record_export_artifacts( + all_years=options["all_years"] + ) rendered = json.dumps( { "generation_id": generation.generation_id, diff --git a/src/organizations/source_record_export.py b/src/organizations/source_record_export.py index 0fbc1b7..df6c9c2 100644 --- a/src/organizations/source_record_export.py +++ b/src/organizations/source_record_export.py @@ -127,7 +127,7 @@ class SourceRecordExportGeneration: generation_id: str generated_at: str - export_year: int + export_year: int | None artifacts: tuple[SourceRecordExportArtifact, ...] records_count: int @@ -197,16 +197,19 @@ def build_source_record_export_artifacts( *, now: datetime | None = None, export_directory: str | Path | None = None, + all_years: bool = False, ) -> SourceRecordExportGeneration: - """Build all files on disk and atomically publish a new generation.""" + """Build annual or all-years files and atomically publish their generation.""" root_directory = _resolve_export_directory(export_directory) + if all_years: + root_directory /= "all-years" generations_directory = root_directory / GENERATION_DIRECTORY_NAME root_directory.mkdir(parents=True, exist_ok=True) generations_directory.mkdir(parents=True, exist_ok=True) generated_at_datetime = _normalize_generation_datetime(now or timezone.now()) - export_year = _export_year(generated_at_datetime) + export_year = None if all_years else _export_year(generated_at_datetime) generation_id = ( f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}" ) @@ -545,7 +548,7 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str def _source_group_queryset( source_group: str, *, - export_year: int, + export_year: int | None, ) -> QuerySet[OrganizationSourceRecord]: queryset = ( OrganizationSourceRecord.objects.filter(extension__source_group=source_group) @@ -555,6 +558,8 @@ def _source_group_queryset( # source groups with millions of rows. .order_by() ) + if export_year is None: + return queryset if source_group == FINANCIAL_SOURCE_GROUP: return queryset.filter(financial_lines__year=export_year).distinct() @@ -581,7 +586,7 @@ def _iter_source_records( *, source_group: str, include_financial_lines: bool, - export_year: int, + export_year: int | None, ) -> Iterator[OrganizationSourceRecord]: iterator = _source_group_queryset( source_group, @@ -597,7 +602,7 @@ def _iter_source_records( Prefetch( "financial_lines", queryset=OrganizationSourceFinancialLine.objects.filter( - year=export_year + **({"year": export_year} if export_year is not None else {}) ).order_by(), ), ) @@ -608,7 +613,7 @@ def _spool_source_group_rows( *, source_group: str, output_path: Path, - export_year: int, + export_year: int | None, ) -> tuple[list[str], int]: include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP payload_headers: set[str] = set() @@ -953,6 +958,9 @@ def _generation_manifest_payload( "generation_id": generation.generation_id, "generated_at": generation.generated_at, "export_year": generation.export_year, + "export_scope": "all_years" + if generation.export_year is None + else "current_year", "records_count": generation.records_count, "artifacts_count": generation.artifacts_count, "files_count": generation.files_count, @@ -973,6 +981,19 @@ def _generation_manifest_payload( } +def _manifest_export_year(payload: dict[str, Any]) -> int | None: + raw_year = payload["export_year"] + scope = payload.get("export_scope", "current_year") + if scope == "all_years" and raw_year is None: + return None + if scope != "current_year" or raw_year is None or isinstance(raw_year, bool): + raise ValueError("Source-record export scope is invalid.") + year = int(raw_year) + if not 1 <= year <= 9999: + raise ValueError("Source-record export year is invalid.") + return year + + def _generation_from_manifest( payload: Any, *, @@ -984,9 +1005,7 @@ def _generation_from_manifest( generation_id = str(payload["generation_id"]) generated_at = str(payload["generated_at"]) datetime.fromisoformat(generated_at) - export_year = int(payload["export_year"]) - if not 1 <= export_year <= 9999: - raise ValueError("Source-record export year is invalid.") + export_year = _manifest_export_year(payload) records_count = int(payload["records_count"]) artifact_payloads = payload["artifacts"] if not isinstance(artifact_payloads, list): diff --git a/src/organizations/tasks.py b/src/organizations/tasks.py index 73efa83..331238c 100644 --- a/src/organizations/tasks.py +++ b/src/organizations/tasks.py @@ -20,6 +20,16 @@ logger = logging.getLogger(__name__) @shared_task(bind=True, base=CorePeriodicTask) def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 """Build and atomically publish the nightly source-record export matrix.""" + return _refresh_source_record_export_artifacts(self, all_years=False) + + +@shared_task(bind=True, base=CorePeriodicTask) +def refresh_all_years_source_record_export_artifacts(self) -> dict: + """Publish all available years independently of the annual generation.""" + return _refresh_source_record_export_artifacts(self, all_years=True) + + +def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict: lock_key = getattr( settings, "SOURCE_RECORD_EXPORT_LOCK_KEY", @@ -29,11 +39,13 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60) ) if not cache.add(lock_key, "1", timeout=lock_ttl): + if all_years: + raise task.retry(countdown=60, max_retries=lock_ttl // 60 + 2) logger.info("Source-record export generation skipped: lock is already held") return {"status": "skipped", "reason": "locked"} try: - generation = build_source_record_export_artifacts() + generation = build_source_record_export_artifacts(all_years=all_years) result = { "status": "success", "generation_id": generation.generation_id, diff --git a/tests/apps/organizations/test_source_record_export.py b/tests/apps/organizations/test_source_record_export.py index d6565c1..f68404b 100644 --- a/tests/apps/organizations/test_source_record_export.py +++ b/tests/apps/organizations/test_source_record_export.py @@ -216,6 +216,126 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase): {export_year}, ) + def test_all_years_generation_is_separate_and_keeps_historical_financial_lines( + self, + ): + organization = Organization.objects.create(name="Export", inn="7707083899") + extension = PlannedInspectionExtension.objects.create(organization=organization) + for year in (2020, 2026): + OrganizationSourceRecord.objects.create( + extension=extension, + record_type="inspection", + source="inspections", + external_id=str(year), + record_date=f"{year}-01-01", + ) + finance = FinancialIndicatorsExtension.objects.create(organization=organization) + report = OrganizationSourceRecord.objects.create( + extension=finance, + record_type="financial_report", + source="fns_reports", + external_id="full", + ) + for year in (2020, 2026): + OrganizationSourceFinancialLine.objects.create( + source_record=report, + year=year, + form_code="1", + line_code=str(year), + ) + source = SourceGroup.PLANNED_INSPECTIONS.value + annual = build_source_record_export_artifacts( + now=datetime(2026, 8, 4, tzinfo=UTC) + ) + annual_manifest = Path(self.export_directory.name) / "current.json" + before = annual_manifest.read_bytes() + full = build_source_record_export_artifacts( + now=datetime(2026, 8, 4, tzinfo=UTC), + all_years=True, + ) + self.assertIsNone(full.export_year) + self.assertEqual(annual_manifest.read_bytes(), before) + full_root = Path(self.export_directory.name) / "all-years" + loaded = load_current_source_record_export_generation( + export_directory=full_root + ) + self.assertEqual(loaded.generation_id, full.generation_id) + self.assertIsNone(loaded.export_year) + + def rows(generation, group): + artifact = next( + a + for a in generation.artifacts + if a.source_group == group and a.file_format == "json" + ) + return _source_records(json.loads(artifact.path.read_text()), group) + + self.assertEqual(len(rows(annual, source)), 1) + self.assertEqual(len(rows(full, source)), 2) + self.assertEqual( + { + line["year"] + for line in rows(full, "financial_indicators")[0]["financial_lines"] + }, + {2020, 2026}, + ) + self.assertEqual( + { + line["year"] + for line in rows(annual, "financial_indicators")[0]["financial_lines"] + }, + {2026}, + ) + # Retention in either scope must not delete the other's files. + for _ in range(3): + build_source_record_export_artifacts(all_years=True) + self.assertTrue(all(a.path.is_file() for a in annual.artifacts)) + + def test_all_years_command_does_not_publish_an_annual_manifest(self): + call_command("build_source_record_exports", all_years=True, stdout=StringIO()) + self.assertFalse((Path(self.export_directory.name) / "current.json").exists()) + full = load_current_source_record_export_generation( + export_directory=Path(self.export_directory.name) / "all-years", + ) + self.assertIsNone(full.export_year) + + def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self): + from celery.exceptions import Retry + from django.core.cache import cache + from organizations.tasks import refresh_all_years_source_record_export_artifacts + + lock = "test-full-export-lock" + with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock): + cache.delete(lock) + cache.add(lock, "held", timeout=30) + try: + with self.assertRaises(Retry): + refresh_all_years_source_record_export_artifacts.run() + self.assertFalse( + ( + Path(self.export_directory.name) / "all-years/current.json" + ).exists() + ) + finally: + cache.delete(lock) + result = refresh_all_years_source_record_export_artifacts.run() + self.assertEqual(result["status"], "success") + self.assertIsNone(result["export_year"]) + self.assertIsNone(cache.get(lock)) + self.assertFalse( + (Path(self.export_directory.name) / "current.json").exists() + ) + + def test_manifest_rejects_ambiguous_all_years_scope(self): + build_source_record_export_artifacts(all_years=True) + directory = Path(self.export_directory.name) / "all-years" + manifest_path = directory / "current.json" + payload = json.loads(manifest_path.read_text()) + payload["export_year"] = 2026 + manifest_path.write_text(json.dumps(payload)) + with self.assertRaises(SourceRecordExportArtifactsUnavailable): + load_current_source_record_export_generation(export_directory=directory) + def test_new_calendar_year_requires_a_new_prepared_generation(self): generated_at = datetime(2026, 12, 31, 23, 59, tzinfo=UTC) build_source_record_export_artifacts(now=generated_at)