feat: add separate all-years source export task

This commit is contained in:
Aleksandr Meshchryakov
2026-09-28 12:06:09 +02:00
parent 8f0c4fe22c
commit 6b43a14edf
5 changed files with 176 additions and 13 deletions

View File

@@ -432,6 +432,115 @@ class SourceRecordExportApiTest(APITestCase):
{export_year},
)
def test_all_years_generation_is_separate_and_keeps_historical_financial_lines(
self,
):
organization = OrganizationFactory.create()
for year in (2020, 2026):
PublicProcurementFactory.create(
organization=organization,
contract_date=datetime(year, 1, 1, tzinfo=UTC).date(),
)
report = FinancialReportFactory.create(organization=organization)
for year in (2020, 2026):
FinancialReportLineFactory.create(
report=report, year=year, line_code=str(year)
)
source = "government_procurements"
annual = build_source_record_export_artifacts(
now=datetime(2026, 8, 4, tzinfo=UTC)
)
annual_manifest = Path(self.export_directory.name) / "current.json"
before = annual_manifest.read_bytes()
full = build_source_record_export_artifacts(
now=datetime(2026, 8, 4, tzinfo=UTC),
all_years=True,
)
self.assertIsNone(full.export_year)
self.assertEqual(annual_manifest.read_bytes(), before)
full_root = Path(self.export_directory.name) / "all-years"
loaded = load_current_source_record_export_generation(
export_directory=full_root
)
self.assertEqual(loaded.generation_id, full.generation_id)
self.assertIsNone(loaded.export_year)
def rows(generation, group):
artifact = next(
a
for a in generation.artifacts
if a.source_group == group and a.file_format == "json"
)
return _source_records(json.loads(artifact.path.read_text()), group)
self.assertEqual(len(rows(annual, source)), 1)
self.assertEqual(len(rows(full, source)), 2)
self.assertEqual(
{
line["year"]
for line in rows(full, "financial_indicators")[0]["financial_lines"]
},
{2020, 2026},
)
self.assertEqual(
{
line["year"]
for line in rows(annual, "financial_indicators")[0]["financial_lines"]
},
{2026},
)
# Retention in either scope must not delete the other's files.
for _ in range(3):
build_source_record_export_artifacts(all_years=True)
self.assertTrue(all(a.path.is_file() for a in annual.artifacts))
def test_all_years_command_does_not_publish_an_annual_manifest(self):
call_command("build_source_record_exports", all_years=True, stdout=StringIO())
self.assertFalse((Path(self.export_directory.name) / "current.json").exists())
full = load_current_source_record_export_generation(
export_directory=Path(self.export_directory.name) / "all-years",
)
self.assertIsNone(full.export_year)
def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self):
from apps.external_data.tasks import (
refresh_all_years_source_record_export_artifacts,
)
from celery.exceptions import Retry
from django.core.cache import cache
lock = "test-full-export-lock"
with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock):
cache.delete(lock)
cache.add(lock, "held", timeout=30)
try:
with self.assertRaises(Retry):
refresh_all_years_source_record_export_artifacts.run()
self.assertFalse(
(
Path(self.export_directory.name) / "all-years/current.json"
).exists()
)
finally:
cache.delete(lock)
result = refresh_all_years_source_record_export_artifacts.run()
self.assertEqual(result["status"], "success")
self.assertIsNone(result["export_year"])
self.assertIsNone(cache.get(lock))
self.assertFalse(
(Path(self.export_directory.name) / "current.json").exists()
)
def test_manifest_rejects_ambiguous_all_years_scope(self):
build_source_record_export_artifacts(all_years=True)
directory = Path(self.export_directory.name) / "all-years"
manifest_path = directory / "current.json"
payload = json.loads(manifest_path.read_text())
payload["export_year"] = 2026
manifest_path.write_text(json.dumps(payload))
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
load_current_source_record_export_generation(export_directory=directory)
def test_new_calendar_year_requires_a_new_prepared_generation(self):
build_source_record_export_artifacts(
now=datetime(2026, 12, 31, 23, 59, tzinfo=UTC)