Export all available years in a separate scheduled task [skip ci] #no_deploy #no_image

Reviewed hotfix. Manual immutable release with data preservation and runtime verification.
This commit was merged in pull request #26.
This commit is contained in:
avm
2026-09-28 13:20:12 +03:00
5 changed files with 176 additions and 13 deletions

View File

@@ -0,0 +1,15 @@
# Выгрузка за все годы
Годовая задача `apps.external_data.tasks.refresh_source_record_export_artifacts` сохраняет прежнее поведение.
Новая задача `apps.external_data.tasks.refresh_all_years_source_record_export_artifacts` собирает все доступные записи каждого поддерживаемого источника без ограничения по году, включая все финансовые строки. Для источников с заменяемыми snapshot выгружается текущее опубликованное состояние: удалённые прежние snapshot не восстанавливаются.
Результат размещается в `SOURCE_RECORD_EXPORT_DIRECTORY/all-years`. У него отдельные `current.json`, `generations` и очистка старых поколений. Публикация атомарна; незавершённая сборка не заменяет опубликованные файлы. Общий lock с годовой задачей ограничивает одновременную нагрузку. Manifest содержит `export_scope: all_years` и `export_year: null`; годовой manifest — `export_scope: current_year` и числовой год. Старые годовые manifest без `export_scope` продолжают читаться.
Ручной запуск:
```sh
python src/manage.py build_source_record_exports --all-years
```
Периодический запуск настраивается отдельной записью django-celery-beat либо системным таймером на контуре без Beat. Задачу следует ставить после годовой сборки. Имена файлов, группировка JSON по ОКПО и форматы CSV/XLSX/JSON сохраняются. API скачивания продолжает отдавать годовой пакет; доступ к полному пакету предоставляется отдельно.

View File

@@ -14,8 +14,18 @@ class Command(BaseAppCommand):
help = "Формирует готовые CSV/XLSX/JSON выгрузки внешних источников"
use_transaction = False
def add_arguments(self, parser) -> None:
super().add_arguments(parser)
parser.add_argument(
"--all-years",
action="store_true",
help="Все доступные годы в отдельном каталоге all-years",
)
def execute_command(self, *args, **options) -> str:
generation = build_source_record_export_artifacts()
generation = build_source_record_export_artifacts(
all_years=options["all_years"]
)
rendered = json.dumps(
{
"generation_id": generation.generation_id,

View File

@@ -524,7 +524,7 @@ class SourceRecordExportGeneration:
generation_id: str
generated_at: str
export_year: int
export_year: int | None
artifacts: tuple[SourceRecordExportArtifact, ...]
records_count: int
@@ -594,16 +594,19 @@ def build_source_record_export_artifacts(
*,
now: datetime | None = None,
export_directory: str | Path | None = None,
all_years: bool = False,
) -> SourceRecordExportGeneration:
"""Build all files on disk and atomically publish a new generation."""
"""Build annual or all-years files and atomically publish their generation."""
root_directory = _resolve_export_directory(export_directory)
if all_years:
root_directory /= "all-years"
generations_directory = root_directory / GENERATION_DIRECTORY_NAME
root_directory.mkdir(parents=True, exist_ok=True)
generations_directory.mkdir(parents=True, exist_ok=True)
generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
export_year = _export_year(generated_at_datetime)
export_year = None if all_years else _export_year(generated_at_datetime)
generation_id = (
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
)
@@ -943,13 +946,13 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str
def _source_model_queryset(
model_spec: SourceModelExportSpec,
*,
export_year: int,
export_year: int | None,
) -> QuerySet:
queryset = cast(
QuerySet,
model_spec.model.objects.select_related("organization").order_by(),
)
if model_spec.all_history:
if export_year is None or model_spec.all_history:
return queryset
if model_spec.export_year_lookup:
return queryset.filter(
@@ -971,7 +974,7 @@ def _source_model_queryset(
def _iter_source_model_records(
model_spec: SourceModelExportSpec,
*,
export_year: int,
export_year: int | None,
) -> Iterator[Any]:
iterator = _source_model_queryset(
model_spec,
@@ -986,7 +989,7 @@ def _iter_source_model_records(
Prefetch(
related_name,
queryset=FinancialReportLine.objects.filter(
year=export_year
**({"year": export_year} if export_year is not None else {})
).order_by(),
)
if related_name == "lines"
@@ -1019,7 +1022,7 @@ def _spool_source_group_rows(
*,
source_spec: SourceGroupExportSpec,
output_path: Path,
export_year: int,
export_year: int | None,
) -> tuple[list[str], int]:
records_count = 0
with output_path.open("w", encoding="utf-8", newline="") as output:
@@ -1405,6 +1408,9 @@ def _generation_manifest_payload(
"generation_id": generation.generation_id,
"generated_at": generation.generated_at,
"export_year": generation.export_year,
"export_scope": "all_years"
if generation.export_year is None
else "current_year",
"records_count": generation.records_count,
"artifacts_count": generation.artifacts_count,
"files_count": generation.files_count,
@@ -1425,6 +1431,19 @@ def _generation_manifest_payload(
}
def _manifest_export_year(payload: dict[str, Any]) -> int | None:
raw_year = payload["export_year"]
scope = payload.get("export_scope", "current_year")
if scope == "all_years" and raw_year is None:
return None
if scope != "current_year" or raw_year is None or isinstance(raw_year, bool):
raise ValueError("Source-record export scope is invalid.")
year = int(raw_year)
if not 1 <= year <= 9999:
raise ValueError("Source-record export year is invalid.")
return year
def _generation_from_manifest(
payload: Any,
*,
@@ -1436,9 +1455,7 @@ def _generation_from_manifest(
generation_id = str(payload["generation_id"])
generated_at = str(payload["generated_at"])
datetime.fromisoformat(generated_at)
export_year = int(payload["export_year"])
if not 1 <= export_year <= 9999:
raise ValueError("Source-record export year is invalid.")
export_year = _manifest_export_year(payload)
records_count = int(payload["records_count"])
artifact_payloads = payload["artifacts"]
if not isinstance(artifact_payloads, list):

View File

@@ -17,6 +17,16 @@ logger = logging.getLogger(__name__)
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
"""Build and atomically publish the nightly external-data export matrix."""
return _refresh_source_record_export_artifacts(self, all_years=False)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_all_years_source_record_export_artifacts(self) -> dict:
"""Publish all available years independently of the annual generation."""
return _refresh_source_record_export_artifacts(self, all_years=True)
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
lock_key = getattr(
settings,
"SOURCE_RECORD_EXPORT_LOCK_KEY",
@@ -26,11 +36,13 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
)
if not cache.add(lock_key, "1", timeout=lock_ttl):
if all_years:
raise task.retry(countdown=60, max_retries=lock_ttl // 60 + 2)
logger.info("Source-record export generation skipped: lock is already held")
return {"status": "skipped", "reason": "locked"}
try:
generation = build_source_record_export_artifacts()
generation = build_source_record_export_artifacts(all_years=all_years)
result = {
"status": "success",
"generation_id": generation.generation_id,

View File

@@ -432,6 +432,115 @@ class SourceRecordExportApiTest(APITestCase):
{export_year},
)
def test_all_years_generation_is_separate_and_keeps_historical_financial_lines(
self,
):
organization = OrganizationFactory.create()
for year in (2020, 2026):
PublicProcurementFactory.create(
organization=organization,
contract_date=datetime(year, 1, 1, tzinfo=UTC).date(),
)
report = FinancialReportFactory.create(organization=organization)
for year in (2020, 2026):
FinancialReportLineFactory.create(
report=report, year=year, line_code=str(year)
)
source = "government_procurements"
annual = build_source_record_export_artifacts(
now=datetime(2026, 8, 4, tzinfo=UTC)
)
annual_manifest = Path(self.export_directory.name) / "current.json"
before = annual_manifest.read_bytes()
full = build_source_record_export_artifacts(
now=datetime(2026, 8, 4, tzinfo=UTC),
all_years=True,
)
self.assertIsNone(full.export_year)
self.assertEqual(annual_manifest.read_bytes(), before)
full_root = Path(self.export_directory.name) / "all-years"
loaded = load_current_source_record_export_generation(
export_directory=full_root
)
self.assertEqual(loaded.generation_id, full.generation_id)
self.assertIsNone(loaded.export_year)
def rows(generation, group):
artifact = next(
a
for a in generation.artifacts
if a.source_group == group and a.file_format == "json"
)
return _source_records(json.loads(artifact.path.read_text()), group)
self.assertEqual(len(rows(annual, source)), 1)
self.assertEqual(len(rows(full, source)), 2)
self.assertEqual(
{
line["year"]
for line in rows(full, "financial_indicators")[0]["financial_lines"]
},
{2020, 2026},
)
self.assertEqual(
{
line["year"]
for line in rows(annual, "financial_indicators")[0]["financial_lines"]
},
{2026},
)
# Retention in either scope must not delete the other's files.
for _ in range(3):
build_source_record_export_artifacts(all_years=True)
self.assertTrue(all(a.path.is_file() for a in annual.artifacts))
def test_all_years_command_does_not_publish_an_annual_manifest(self):
call_command("build_source_record_exports", all_years=True, stdout=StringIO())
self.assertFalse((Path(self.export_directory.name) / "current.json").exists())
full = load_current_source_record_export_generation(
export_directory=Path(self.export_directory.name) / "all-years",
)
self.assertIsNone(full.export_year)
def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self):
from apps.external_data.tasks import (
refresh_all_years_source_record_export_artifacts,
)
from celery.exceptions import Retry
from django.core.cache import cache
lock = "test-full-export-lock"
with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock):
cache.delete(lock)
cache.add(lock, "held", timeout=30)
try:
with self.assertRaises(Retry):
refresh_all_years_source_record_export_artifacts.run()
self.assertFalse(
(
Path(self.export_directory.name) / "all-years/current.json"
).exists()
)
finally:
cache.delete(lock)
result = refresh_all_years_source_record_export_artifacts.run()
self.assertEqual(result["status"], "success")
self.assertIsNone(result["export_year"])
self.assertIsNone(cache.get(lock))
self.assertFalse(
(Path(self.export_directory.name) / "current.json").exists()
)
def test_manifest_rejects_ambiguous_all_years_scope(self):
build_source_record_export_artifacts(all_years=True)
directory = Path(self.export_directory.name) / "all-years"
manifest_path = directory / "current.json"
payload = json.loads(manifest_path.read_text())
payload["export_year"] = 2026
manifest_path.write_text(json.dumps(payload))
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
load_current_source_record_export_generation(export_directory=directory)
def test_new_calendar_year_requires_a_new_prepared_generation(self):
build_source_record_export_artifacts(
now=datetime(2026, 12, 31, 23, 59, tzinfo=UTC)