From 0d4743e230051eb0953648292d64ef53e308156e Mon Sep 17 00:00:00 2001 From: Aleksandr Meshchryakov Date: Mon, 28 Sep 2026 12:06:09 +0200 Subject: [PATCH] feat: add separate all-years source export task --- docs/all-years-source-exports.md | 15 +++ .../commands/build_source_record_exports.py | 12 +- .../external_data/source_record_export.py | 39 +++++-- src/apps/external_data/tasks.py | 14 ++- .../test_source_record_export.py | 109 ++++++++++++++++++ 5 files changed, 177 insertions(+), 12 deletions(-) create mode 100644 docs/all-years-source-exports.md diff --git a/docs/all-years-source-exports.md b/docs/all-years-source-exports.md new file mode 100644 index 0000000..77ab914 --- /dev/null +++ b/docs/all-years-source-exports.md @@ -0,0 +1,15 @@ +# Выгрузка за все годы + +Годовая задача `apps.external_data.tasks.refresh_source_record_export_artifacts` сохраняет прежнее поведение. + +Новая задача `apps.external_data.tasks.refresh_all_years_source_record_export_artifacts` собирает все доступные записи каждого поддерживаемого источника без ограничения по году, включая все финансовые строки. Для источников с заменяемыми snapshot выгружается текущее опубликованное состояние: удалённые прежние snapshot не восстанавливаются. + +Результат размещается в `SOURCE_RECORD_EXPORT_DIRECTORY/all-years`. У него отдельные `current.json`, `generations` и очистка старых поколений. Публикация атомарна; незавершённая сборка не заменяет опубликованные файлы. Общий lock с годовой задачей ограничивает одновременную нагрузку. Manifest содержит `export_scope: all_years` и `export_year: null`; годовой manifest — `export_scope: current_year` и числовой год. Старые годовые manifest без `export_scope` продолжают читаться. + +Ручной запуск: + +```sh +python src/manage.py build_source_record_exports --all-years +``` + +Периодический запуск настраивается отдельной записью django-celery-beat либо системным таймером на контуре без Beat. Задачу следует ставить после годовой сборки. Имена файлов, группировка JSON по ОКПО и форматы CSV/XLSX/JSON сохраняются. API скачивания продолжает отдавать годовой пакет; доступ к полному пакету предоставляется отдельно. diff --git a/src/apps/external_data/management/commands/build_source_record_exports.py b/src/apps/external_data/management/commands/build_source_record_exports.py index 1d8c4ef..9b0a0c8 100644 --- a/src/apps/external_data/management/commands/build_source_record_exports.py +++ b/src/apps/external_data/management/commands/build_source_record_exports.py @@ -14,8 +14,18 @@ class Command(BaseAppCommand): help = "Формирует готовые CSV/XLSX/JSON выгрузки внешних источников" use_transaction = False + def add_arguments(self, parser) -> None: + super().add_arguments(parser) + parser.add_argument( + "--all-years", + action="store_true", + help="Все доступные годы в отдельном каталоге all-years", + ) + def execute_command(self, *args, **options) -> str: - generation = build_source_record_export_artifacts() + generation = build_source_record_export_artifacts( + all_years=options["all_years"] + ) rendered = json.dumps( { "generation_id": generation.generation_id, diff --git a/src/apps/external_data/source_record_export.py b/src/apps/external_data/source_record_export.py index de645c1..1b193b7 100644 --- a/src/apps/external_data/source_record_export.py +++ b/src/apps/external_data/source_record_export.py @@ -442,7 +442,7 @@ class SourceRecordExportGeneration: generation_id: str generated_at: str - export_year: int + export_year: int | None artifacts: tuple[SourceRecordExportArtifact, ...] records_count: int @@ -512,16 +512,19 @@ def build_source_record_export_artifacts( *, now: datetime | None = None, export_directory: str | Path | None = None, + all_years: bool = False, ) -> SourceRecordExportGeneration: - """Build all files on disk and atomically publish a new generation.""" + """Build annual or all-years files and atomically publish their generation.""" root_directory = _resolve_export_directory(export_directory) + if all_years: + root_directory /= "all-years" generations_directory = root_directory / GENERATION_DIRECTORY_NAME root_directory.mkdir(parents=True, exist_ok=True) generations_directory.mkdir(parents=True, exist_ok=True) generated_at_datetime = _normalize_generation_datetime(now or timezone.now()) - export_year = _export_year(generated_at_datetime) + export_year = None if all_years else _export_year(generated_at_datetime) generation_id = ( f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}" ) @@ -861,12 +864,14 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str def _source_model_queryset( model_spec: SourceModelExportSpec, *, - export_year: int, + export_year: int | None, ) -> QuerySet: queryset = cast( QuerySet, model_spec.model.objects.select_related("organization").order_by(), ) + if export_year is None: + return queryset if model_spec.export_year_lookup: return queryset.filter( **{model_spec.export_year_lookup: export_year} @@ -887,7 +892,7 @@ def _source_model_queryset( def _iter_source_model_records( model_spec: SourceModelExportSpec, *, - export_year: int, + export_year: int | None, ) -> Iterator[Any]: iterator = _source_model_queryset( model_spec, @@ -902,7 +907,7 @@ def _iter_source_model_records( Prefetch( related_name, queryset=FinancialReportLine.objects.filter( - year=export_year + **({"year": export_year} if export_year is not None else {}) ).order_by(), ) if related_name == "lines" @@ -935,7 +940,7 @@ def _spool_source_group_rows( *, source_spec: SourceGroupExportSpec, output_path: Path, - export_year: int, + export_year: int | None, ) -> tuple[list[str], int]: records_count = 0 with output_path.open("w", encoding="utf-8", newline="") as output: @@ -1316,6 +1321,9 @@ def _generation_manifest_payload( "generation_id": generation.generation_id, "generated_at": generation.generated_at, "export_year": generation.export_year, + "export_scope": "all_years" + if generation.export_year is None + else "current_year", "records_count": generation.records_count, "artifacts_count": generation.artifacts_count, "files_count": generation.files_count, @@ -1336,6 +1344,19 @@ def _generation_manifest_payload( } +def _manifest_export_year(payload: dict[str, Any]) -> int | None: + raw_year = payload["export_year"] + scope = payload.get("export_scope", "current_year") + if scope == "all_years" and raw_year is None: + return None + if scope != "current_year" or raw_year is None or isinstance(raw_year, bool): + raise ValueError("Source-record export scope is invalid.") + year = int(raw_year) + if not 1 <= year <= 9999: + raise ValueError("Source-record export year is invalid.") + return year + + def _generation_from_manifest( payload: Any, *, @@ -1347,9 +1368,7 @@ def _generation_from_manifest( generation_id = str(payload["generation_id"]) generated_at = str(payload["generated_at"]) datetime.fromisoformat(generated_at) - export_year = int(payload["export_year"]) - if not 1 <= export_year <= 9999: - raise ValueError("Source-record export year is invalid.") + export_year = _manifest_export_year(payload) records_count = int(payload["records_count"]) artifact_payloads = payload["artifacts"] if not isinstance(artifact_payloads, list): diff --git a/src/apps/external_data/tasks.py b/src/apps/external_data/tasks.py index 8950260..13ca0ac 100644 --- a/src/apps/external_data/tasks.py +++ b/src/apps/external_data/tasks.py @@ -17,6 +17,16 @@ logger = logging.getLogger(__name__) def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 """Build and atomically publish the nightly external-data export matrix.""" + return _refresh_source_record_export_artifacts(self, all_years=False) + + +@shared_task(bind=True, base=CorePeriodicTask) +def refresh_all_years_source_record_export_artifacts(self) -> dict: + """Publish all available years independently of the annual generation.""" + return _refresh_source_record_export_artifacts(self, all_years=True) + + +def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict: lock_key = getattr( settings, "SOURCE_RECORD_EXPORT_LOCK_KEY", @@ -26,11 +36,13 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60) ) if not cache.add(lock_key, "1", timeout=lock_ttl): + if all_years: + raise task.retry(countdown=60, max_retries=lock_ttl // 60 + 2) logger.info("Source-record export generation skipped: lock is already held") return {"status": "skipped", "reason": "locked"} try: - generation = build_source_record_export_artifacts() + generation = build_source_record_export_artifacts(all_years=all_years) result = { "status": "success", "generation_id": generation.generation_id, diff --git a/tests/apps/external_data/test_source_record_export.py b/tests/apps/external_data/test_source_record_export.py index e39d7bd..3101481 100644 --- a/tests/apps/external_data/test_source_record_export.py +++ b/tests/apps/external_data/test_source_record_export.py @@ -396,6 +396,115 @@ class SourceRecordExportApiTest(APITestCase): {export_year}, ) + def test_all_years_generation_is_separate_and_keeps_historical_financial_lines( + self, + ): + organization = OrganizationFactory.create() + for year in (2020, 2026): + PublicProcurementFactory.create( + organization=organization, + contract_date=datetime(year, 1, 1, tzinfo=UTC).date(), + ) + report = FinancialReportFactory.create(organization=organization) + for year in (2020, 2026): + FinancialReportLineFactory.create( + report=report, year=year, line_code=str(year) + ) + source = "government_procurements" + annual = build_source_record_export_artifacts( + now=datetime(2026, 8, 4, tzinfo=UTC) + ) + annual_manifest = Path(self.export_directory.name) / "current.json" + before = annual_manifest.read_bytes() + full = build_source_record_export_artifacts( + now=datetime(2026, 8, 4, tzinfo=UTC), + all_years=True, + ) + self.assertIsNone(full.export_year) + self.assertEqual(annual_manifest.read_bytes(), before) + full_root = Path(self.export_directory.name) / "all-years" + loaded = load_current_source_record_export_generation( + export_directory=full_root + ) + self.assertEqual(loaded.generation_id, full.generation_id) + self.assertIsNone(loaded.export_year) + + def rows(generation, group): + artifact = next( + a + for a in generation.artifacts + if a.source_group == group and a.file_format == "json" + ) + return _source_records(json.loads(artifact.path.read_text()), group) + + self.assertEqual(len(rows(annual, source)), 1) + self.assertEqual(len(rows(full, source)), 2) + self.assertEqual( + { + line["year"] + for line in rows(full, "financial_indicators")[0]["financial_lines"] + }, + {2020, 2026}, + ) + self.assertEqual( + { + line["year"] + for line in rows(annual, "financial_indicators")[0]["financial_lines"] + }, + {2026}, + ) + # Retention in either scope must not delete the other's files. + for _ in range(3): + build_source_record_export_artifacts(all_years=True) + self.assertTrue(all(a.path.is_file() for a in annual.artifacts)) + + def test_all_years_command_does_not_publish_an_annual_manifest(self): + call_command("build_source_record_exports", all_years=True, stdout=StringIO()) + self.assertFalse((Path(self.export_directory.name) / "current.json").exists()) + full = load_current_source_record_export_generation( + export_directory=Path(self.export_directory.name) / "all-years", + ) + self.assertIsNone(full.export_year) + + def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self): + from apps.external_data.tasks import ( + refresh_all_years_source_record_export_artifacts, + ) + from celery.exceptions import Retry + from django.core.cache import cache + + lock = "test-full-export-lock" + with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock): + cache.delete(lock) + cache.add(lock, "held", timeout=30) + try: + with self.assertRaises(Retry): + refresh_all_years_source_record_export_artifacts.run() + self.assertFalse( + ( + Path(self.export_directory.name) / "all-years/current.json" + ).exists() + ) + finally: + cache.delete(lock) + result = refresh_all_years_source_record_export_artifacts.run() + self.assertEqual(result["status"], "success") + self.assertIsNone(result["export_year"]) + self.assertIsNone(cache.get(lock)) + self.assertFalse( + (Path(self.export_directory.name) / "current.json").exists() + ) + + def test_manifest_rejects_ambiguous_all_years_scope(self): + build_source_record_export_artifacts(all_years=True) + directory = Path(self.export_directory.name) / "all-years" + manifest_path = directory / "current.json" + payload = json.loads(manifest_path.read_text()) + payload["export_year"] = 2026 + manifest_path.write_text(json.dumps(payload)) + with self.assertRaises(SourceRecordExportArtifactsUnavailable): + load_current_source_record_export_generation(export_directory=directory) + def test_new_calendar_year_requires_a_new_prepared_generation(self): build_source_record_export_artifacts( now=datetime(2026, 12, 31, 23, 59, tzinfo=UTC)