feat: add separate all-years source export task
This commit is contained in:
@@ -432,6 +432,115 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
{export_year},
|
||||
)
|
||||
|
||||
def test_all_years_generation_is_separate_and_keeps_historical_financial_lines(
|
||||
self,
|
||||
):
|
||||
organization = OrganizationFactory.create()
|
||||
for year in (2020, 2026):
|
||||
PublicProcurementFactory.create(
|
||||
organization=organization,
|
||||
contract_date=datetime(year, 1, 1, tzinfo=UTC).date(),
|
||||
)
|
||||
report = FinancialReportFactory.create(organization=organization)
|
||||
for year in (2020, 2026):
|
||||
FinancialReportLineFactory.create(
|
||||
report=report, year=year, line_code=str(year)
|
||||
)
|
||||
source = "government_procurements"
|
||||
annual = build_source_record_export_artifacts(
|
||||
now=datetime(2026, 8, 4, tzinfo=UTC)
|
||||
)
|
||||
annual_manifest = Path(self.export_directory.name) / "current.json"
|
||||
before = annual_manifest.read_bytes()
|
||||
full = build_source_record_export_artifacts(
|
||||
now=datetime(2026, 8, 4, tzinfo=UTC),
|
||||
all_years=True,
|
||||
)
|
||||
self.assertIsNone(full.export_year)
|
||||
self.assertEqual(annual_manifest.read_bytes(), before)
|
||||
full_root = Path(self.export_directory.name) / "all-years"
|
||||
loaded = load_current_source_record_export_generation(
|
||||
export_directory=full_root
|
||||
)
|
||||
self.assertEqual(loaded.generation_id, full.generation_id)
|
||||
self.assertIsNone(loaded.export_year)
|
||||
|
||||
def rows(generation, group):
|
||||
artifact = next(
|
||||
a
|
||||
for a in generation.artifacts
|
||||
if a.source_group == group and a.file_format == "json"
|
||||
)
|
||||
return _source_records(json.loads(artifact.path.read_text()), group)
|
||||
|
||||
self.assertEqual(len(rows(annual, source)), 1)
|
||||
self.assertEqual(len(rows(full, source)), 2)
|
||||
self.assertEqual(
|
||||
{
|
||||
line["year"]
|
||||
for line in rows(full, "financial_indicators")[0]["financial_lines"]
|
||||
},
|
||||
{2020, 2026},
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
line["year"]
|
||||
for line in rows(annual, "financial_indicators")[0]["financial_lines"]
|
||||
},
|
||||
{2026},
|
||||
)
|
||||
# Retention in either scope must not delete the other's files.
|
||||
for _ in range(3):
|
||||
build_source_record_export_artifacts(all_years=True)
|
||||
self.assertTrue(all(a.path.is_file() for a in annual.artifacts))
|
||||
|
||||
def test_all_years_command_does_not_publish_an_annual_manifest(self):
|
||||
call_command("build_source_record_exports", all_years=True, stdout=StringIO())
|
||||
self.assertFalse((Path(self.export_directory.name) / "current.json").exists())
|
||||
full = load_current_source_record_export_generation(
|
||||
export_directory=Path(self.export_directory.name) / "all-years",
|
||||
)
|
||||
self.assertIsNone(full.export_year)
|
||||
|
||||
def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self):
|
||||
from apps.external_data.tasks import (
|
||||
refresh_all_years_source_record_export_artifacts,
|
||||
)
|
||||
from celery.exceptions import Retry
|
||||
from django.core.cache import cache
|
||||
|
||||
lock = "test-full-export-lock"
|
||||
with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock):
|
||||
cache.delete(lock)
|
||||
cache.add(lock, "held", timeout=30)
|
||||
try:
|
||||
with self.assertRaises(Retry):
|
||||
refresh_all_years_source_record_export_artifacts.run()
|
||||
self.assertFalse(
|
||||
(
|
||||
Path(self.export_directory.name) / "all-years/current.json"
|
||||
).exists()
|
||||
)
|
||||
finally:
|
||||
cache.delete(lock)
|
||||
result = refresh_all_years_source_record_export_artifacts.run()
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertIsNone(result["export_year"])
|
||||
self.assertIsNone(cache.get(lock))
|
||||
self.assertFalse(
|
||||
(Path(self.export_directory.name) / "current.json").exists()
|
||||
)
|
||||
|
||||
def test_manifest_rejects_ambiguous_all_years_scope(self):
|
||||
build_source_record_export_artifacts(all_years=True)
|
||||
directory = Path(self.export_directory.name) / "all-years"
|
||||
manifest_path = directory / "current.json"
|
||||
payload = json.loads(manifest_path.read_text())
|
||||
payload["export_year"] = 2026
|
||||
manifest_path.write_text(json.dumps(payload))
|
||||
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
|
||||
load_current_source_record_export_generation(export_directory=directory)
|
||||
|
||||
def test_new_calendar_year_requires_a_new_prepared_generation(self):
|
||||
build_source_record_export_artifacts(
|
||||
now=datetime(2026, 12, 31, 23, 59, tzinfo=UTC)
|
||||
|
||||
Reference in New Issue
Block a user