feat: partition all source exports by year and enable nightly builds
This commit is contained in:
@@ -369,8 +369,20 @@ class SourceExportCatalogTest(APITestCase):
|
||||
rows = json.loads(self.bytes(consume_ticket(ticket["ticket"])))
|
||||
self.assertEqual(
|
||||
[r["year"] for r in rows[0]["financial_indicators"][0]["financial_lines"]],
|
||||
[2022, 2023],
|
||||
[2022],
|
||||
)
|
||||
self.assertEqual(generation.manifest["records_count"], 1)
|
||||
self.assertEqual(generation.manifest["exported_records_count"], 2)
|
||||
self.assertEqual(
|
||||
[d["year"] for d in generation.manifest["datasets"]], [2022, 2023]
|
||||
)
|
||||
for part in generation.manifest["parts"]:
|
||||
variant = part["variants"][0]
|
||||
projected = json.loads(
|
||||
(_directory(generation) / variant["file_name"]).read_bytes()
|
||||
)
|
||||
lines = projected[0]["financial_indicators"][0]["financial_lines"]
|
||||
self.assertEqual({line["year"] for line in lines}, {part["year"]})
|
||||
with self.assertRaises(ExportError) as error:
|
||||
issue_ticket(self.user, self.selector(generation, fmt="xlsx"))
|
||||
self.assertEqual(error.exception.status, 409)
|
||||
@@ -394,14 +406,17 @@ class SourceExportCatalogTest(APITestCase):
|
||||
import secrets
|
||||
|
||||
for record in self.records(5):
|
||||
record.record_date = "2024-01-01"
|
||||
record.payload = {"text": secrets.token_hex(1200)}
|
||||
record.save(update_fields=["payload"])
|
||||
record.save(update_fields=["payload", "record_date"])
|
||||
with override_settings(
|
||||
SOURCE_EXPORT_ROWS_PER_PART=100, SOURCE_EXPORT_MAX_FILE_BYTES=10000
|
||||
):
|
||||
generation = build_source(GROUP)
|
||||
self.assertGreater(len(generation.manifest["parts"]), 1)
|
||||
for part in generation.manifest["parts"]:
|
||||
self.assertEqual(part["year"], 2024)
|
||||
self.assertEqual(part["dataset_key"], "records_2024")
|
||||
self.assertTrue(all(v["size_bytes"] <= 10000 for v in part["variants"]))
|
||||
self.assertEqual(
|
||||
sum(p["records_count"] for p in generation.manifest["parts"]), 5
|
||||
@@ -681,3 +696,228 @@ class SourceExportCatalogTest(APITestCase):
|
||||
self.base + "export-ticket/", "{bad", content_type="application/json"
|
||||
)
|
||||
self.assertEqual(response.status_code, 401)
|
||||
|
||||
def test_year_boundaries_dates_and_undated_do_not_mix(self):
|
||||
from datetime import UTC, datetime
|
||||
|
||||
records = self.records(5)
|
||||
values = ["2021-12-31", "01.01.2022", "", "неизвестно", "16599930000"]
|
||||
for record, value in zip(records, values, strict=True):
|
||||
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
|
||||
record_date=value, created_at=datetime(2023, 12, 31, 22, 0, tzinfo=UTC)
|
||||
)
|
||||
with override_settings(
|
||||
TIME_ZONE="Europe/Moscow", SOURCE_EXPORT_ROWS_PER_PART=1
|
||||
), timezone.override("Pacific/Honolulu"):
|
||||
generation = build_source(GROUP)
|
||||
by_year = {}
|
||||
for part in generation.manifest["parts"]:
|
||||
variant = next(v for v in part["variants"] if v["format"] == "json")
|
||||
rows = json.loads(
|
||||
(_directory(generation) / variant["file_name"]).read_bytes()
|
||||
)
|
||||
by_year.setdefault(part["year"], set()).update(
|
||||
r["uid"] for org in rows for r in org[GROUP]
|
||||
)
|
||||
self.assertEqual(
|
||||
by_year,
|
||||
{
|
||||
2021: {str(records[0].uid)},
|
||||
2022: {str(records[1].uid)},
|
||||
2024: {str(records[2].uid)},
|
||||
None: {str(records[3].uid), str(records[4].uid)},
|
||||
},
|
||||
)
|
||||
self.assertEqual(generation.manifest["exported_records_count"], 5)
|
||||
self.assertEqual(
|
||||
[d["year"] for d in generation.manifest["datasets"]],
|
||||
[2021, 2022, 2024, None],
|
||||
)
|
||||
|
||||
def test_year_parts_numbers_formats_and_zip_directories(self):
|
||||
records = self.records(7)
|
||||
generation = build_source(GROUP)
|
||||
self.assertEqual(
|
||||
[d["records_count"] for d in generation.manifest["datasets"]], [3, 2, 2]
|
||||
)
|
||||
self.assertEqual(
|
||||
[p["part_number"] for p in generation.manifest["parts"]], [1, 2, 1, 1]
|
||||
)
|
||||
for part in generation.manifest["parts"]:
|
||||
for variant in part["variants"]:
|
||||
self.assertIn(f'-{part["year"]}-part-', variant["file_name"])
|
||||
variant = next(v for v in part["variants"] if v["format"] == "json")
|
||||
rows = json.loads(
|
||||
(_directory(generation) / variant["file_name"]).read_bytes()
|
||||
)
|
||||
projected = [r for org in rows for r in org[GROUP]]
|
||||
self.assertEqual(
|
||||
{int(r["record_date"][:4]) for r in projected}, {part["year"]}
|
||||
)
|
||||
ticket = issue_ticket(
|
||||
self.user, self.selector(generation, "source_archive", "json")
|
||||
)
|
||||
with zipfile.ZipFile(
|
||||
BytesIO(self.bytes(consume_ticket(ticket["ticket"])))
|
||||
) as archive:
|
||||
self.assertEqual(
|
||||
{n.split("/")[0] for n in archive.namelist() if n != "manifest.json"},
|
||||
{"2024", "2025", "2026"},
|
||||
)
|
||||
self.assertEqual(len(archive.namelist()), 5)
|
||||
manifest = json.loads(archive.read("manifest.json"))
|
||||
self.assertEqual(manifest["partitioning"], "year")
|
||||
self.assertEqual(manifest["records_count"], len(records))
|
||||
|
||||
def test_financial_record_without_lines_and_invalid_line_year_are_retained(self):
|
||||
extension = FinancialIndicatorsExtension.objects.create(
|
||||
organization=self.organization
|
||||
)
|
||||
empty = OrganizationSourceRecord.objects.create(
|
||||
extension=extension,
|
||||
source="fns_reports",
|
||||
record_type="financial_report",
|
||||
external_id=str(uuid4()),
|
||||
record_date="2019-01-01",
|
||||
)
|
||||
invalid = OrganizationSourceRecord.objects.create(
|
||||
extension=extension,
|
||||
source="fns_reports",
|
||||
record_type="financial_report",
|
||||
external_id=str(uuid4()),
|
||||
)
|
||||
OrganizationSourceFinancialLine.objects.create(
|
||||
source_record=invalid, year=0, form_code="1", line_code="1"
|
||||
)
|
||||
generation = build_source("financial_indicators")
|
||||
self.assertEqual(
|
||||
[d["year"] for d in generation.manifest["datasets"]], [2019, None]
|
||||
)
|
||||
self.assertEqual(generation.manifest["records_count"], 2)
|
||||
self.assertEqual(generation.manifest["exported_records_count"], 2)
|
||||
rows = []
|
||||
for part in generation.manifest["parts"]:
|
||||
variant = part["variants"][0]
|
||||
rows.extend(
|
||||
r
|
||||
for org in json.loads(
|
||||
(_directory(generation) / variant["file_name"]).read_bytes()
|
||||
)
|
||||
for r in org["financial_indicators"]
|
||||
)
|
||||
self.assertEqual({r["uid"] for r in rows}, {str(empty.uid), str(invalid.uid)})
|
||||
|
||||
def test_pre_partition_generation_metadata_and_ticket_stay_readable(self):
|
||||
self.records(1)
|
||||
generation = build_source(GROUP)
|
||||
manifest = generation.manifest
|
||||
manifest["version"] = 3
|
||||
manifest["description"]["example_dataset_key"] = "records"
|
||||
manifest.pop("partitioning")
|
||||
manifest.pop("exported_records_count")
|
||||
for part in manifest["parts"]:
|
||||
part.pop("year")
|
||||
part["dataset_key"] = "records"
|
||||
for dataset in manifest["datasets"]:
|
||||
dataset.pop("year")
|
||||
dataset["key"] = "records"
|
||||
dataset["export_schema_version"] = "1"
|
||||
generation.save(update_fields=["manifest"])
|
||||
ticket = issue_ticket(self.user, self.selector(generation))
|
||||
expected = (_directory(generation) / ticket["file_name"]).read_bytes()
|
||||
build_source(GROUP)
|
||||
data, _ = detail(GROUP, QueryDict(f"generation_id={generation.pk}"))
|
||||
self.assertEqual(data["partitioning"], "none")
|
||||
self.assertIsNone(data["files"][0]["year"])
|
||||
self.assertIsNone(data["datasets"][0]["year"])
|
||||
self.assertEqual(self.bytes(consume_ticket(ticket["ticket"])), expected)
|
||||
|
||||
def test_nightly_dispatch_builds_twenty_two_sources_for_both_years(self):
|
||||
from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS
|
||||
from organizations.tasks import refresh_source_export_catalog
|
||||
|
||||
for group in registry():
|
||||
descriptor = next(
|
||||
d for d in SOURCE_GROUP_DESCRIPTORS.values() if d.source_group == group
|
||||
)
|
||||
extension = descriptor.extension_model.objects.filter(
|
||||
organization=self.organization
|
||||
).first() or descriptor.extension_model.objects.create(
|
||||
organization=self.organization
|
||||
)
|
||||
for year in [2020, 2021]:
|
||||
record = OrganizationSourceRecord.objects.create(
|
||||
extension=extension,
|
||||
source=descriptor.source,
|
||||
record_type=descriptor.record_type,
|
||||
external_id=str(uuid4()),
|
||||
record_date=f"{year}-01-01",
|
||||
)
|
||||
if group == "financial_indicators":
|
||||
OrganizationSourceFinancialLine.objects.create(
|
||||
source_record=record, year=year, form_code="1", line_code="1"
|
||||
)
|
||||
with patch.object(
|
||||
refresh_source_export_catalog,
|
||||
"delay",
|
||||
side_effect=lambda **kw: refresh_source_export_catalog.run(**kw),
|
||||
):
|
||||
result = refresh_source_export_catalog.run()
|
||||
self.assertEqual(result["groups"], 22)
|
||||
self.assertEqual(SourceExportGeneration.objects.count(), 22)
|
||||
for generation in SourceExportGeneration.objects.all():
|
||||
self.assertEqual(generation.manifest["partitioning"], "year")
|
||||
self.assertEqual(
|
||||
[d["year"] for d in generation.manifest["datasets"]], [2020, 2021]
|
||||
)
|
||||
self.assertEqual(generation.manifest["records_count"], 2)
|
||||
|
||||
def test_nightly_migration_enables_recurring_all_sources_schedule(self):
|
||||
from importlib import import_module
|
||||
|
||||
from django.apps import apps
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
|
||||
migration = import_module(
|
||||
"organizations.migrations.0021_enable_yearly_partitioned_exports"
|
||||
)
|
||||
migration.enable(apps, None)
|
||||
task = PeriodicTask.objects.get(name=migration.NAME)
|
||||
self.assertTrue(task.enabled)
|
||||
self.assertFalse(task.one_off)
|
||||
self.assertEqual(task.kwargs, "{}")
|
||||
self.assertEqual(task.crontab.hour, "6")
|
||||
self.assertEqual(task.crontab.minute, "30")
|
||||
self.assertEqual(str(task.crontab.timezone), "Europe/Moscow")
|
||||
task.one_off = True
|
||||
task.expires = timezone.now() - timedelta(days=1)
|
||||
task.start_time = timezone.now() + timedelta(days=10)
|
||||
task.kwargs = '{"source_group":"arbitration"}'
|
||||
task.save()
|
||||
migration.enable(apps, None)
|
||||
task.refresh_from_db()
|
||||
self.assertFalse(task.one_off)
|
||||
self.assertIsNone(task.expires)
|
||||
self.assertIsNone(task.expire_seconds)
|
||||
self.assertIsNone(task.start_time)
|
||||
self.assertEqual(task.kwargs, "{}")
|
||||
self.assertEqual(PeriodicTask.objects.filter(name=migration.NAME).count(), 1)
|
||||
|
||||
def test_nightly_migration_notifies_beat_without_live_model_signals(self):
|
||||
from importlib import import_module
|
||||
|
||||
from django.apps import apps
|
||||
from django_celery_beat.models import PeriodicTasks
|
||||
|
||||
migration = import_module(
|
||||
"organizations.migrations.0021_enable_yearly_partitioned_exports"
|
||||
)
|
||||
# Historical migration models do not execute the live save hooks.
|
||||
with patch.object(PeriodicTasks, "update_changed"):
|
||||
PeriodicTasks.objects.all().delete()
|
||||
migration.enable(apps, None)
|
||||
self.assertTrue(PeriodicTasks.objects.filter(ident=1).exists())
|
||||
old = timezone.now() - timedelta(days=30)
|
||||
PeriodicTasks.objects.filter(ident=1).update(last_update=old)
|
||||
migration.disable(apps, None)
|
||||
self.assertGreater(PeriodicTasks.objects.get(ident=1).last_update, old)
|
||||
|
||||
Reference in New Issue
Block a user