feat: partition all source exports by year and enable nightly builds

This commit is contained in:
Aleksandr Meshchryakov
2026-10-22 21:45:29 +02:00
parent 63872e86fe
commit 2002e80196
13 changed files with 672 additions and 87 deletions

View File

@@ -369,8 +369,20 @@ class SourceExportCatalogTest(APITestCase):
rows = json.loads(self.bytes(consume_ticket(ticket["ticket"])))
self.assertEqual(
[r["year"] for r in rows[0]["financial_indicators"][0]["financial_lines"]],
[2022, 2023],
[2022],
)
self.assertEqual(generation.manifest["records_count"], 1)
self.assertEqual(generation.manifest["exported_records_count"], 2)
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2022, 2023]
)
for part in generation.manifest["parts"]:
variant = part["variants"][0]
projected = json.loads(
(_directory(generation) / variant["file_name"]).read_bytes()
)
lines = projected[0]["financial_indicators"][0]["financial_lines"]
self.assertEqual({line["year"] for line in lines}, {part["year"]})
with self.assertRaises(ExportError) as error:
issue_ticket(self.user, self.selector(generation, fmt="xlsx"))
self.assertEqual(error.exception.status, 409)
@@ -394,14 +406,17 @@ class SourceExportCatalogTest(APITestCase):
import secrets
for record in self.records(5):
record.record_date = "2024-01-01"
record.payload = {"text": secrets.token_hex(1200)}
record.save(update_fields=["payload"])
record.save(update_fields=["payload", "record_date"])
with override_settings(
SOURCE_EXPORT_ROWS_PER_PART=100, SOURCE_EXPORT_MAX_FILE_BYTES=10000
):
generation = build_source(GROUP)
self.assertGreater(len(generation.manifest["parts"]), 1)
for part in generation.manifest["parts"]:
self.assertEqual(part["year"], 2024)
self.assertEqual(part["dataset_key"], "records_2024")
self.assertTrue(all(v["size_bytes"] <= 10000 for v in part["variants"]))
self.assertEqual(
sum(p["records_count"] for p in generation.manifest["parts"]), 5
@@ -681,3 +696,228 @@ class SourceExportCatalogTest(APITestCase):
self.base + "export-ticket/", "{bad", content_type="application/json"
)
self.assertEqual(response.status_code, 401)
def test_year_boundaries_dates_and_undated_do_not_mix(self):
from datetime import UTC, datetime
records = self.records(5)
values = ["2021-12-31", "01.01.2022", "", "неизвестно", "16599930000"]
for record, value in zip(records, values, strict=True):
OrganizationSourceRecord.objects.filter(pk=record.pk).update(
record_date=value, created_at=datetime(2023, 12, 31, 22, 0, tzinfo=UTC)
)
with override_settings(
TIME_ZONE="Europe/Moscow", SOURCE_EXPORT_ROWS_PER_PART=1
), timezone.override("Pacific/Honolulu"):
generation = build_source(GROUP)
by_year = {}
for part in generation.manifest["parts"]:
variant = next(v for v in part["variants"] if v["format"] == "json")
rows = json.loads(
(_directory(generation) / variant["file_name"]).read_bytes()
)
by_year.setdefault(part["year"], set()).update(
r["uid"] for org in rows for r in org[GROUP]
)
self.assertEqual(
by_year,
{
2021: {str(records[0].uid)},
2022: {str(records[1].uid)},
2024: {str(records[2].uid)},
None: {str(records[3].uid), str(records[4].uid)},
},
)
self.assertEqual(generation.manifest["exported_records_count"], 5)
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]],
[2021, 2022, 2024, None],
)
def test_year_parts_numbers_formats_and_zip_directories(self):
records = self.records(7)
generation = build_source(GROUP)
self.assertEqual(
[d["records_count"] for d in generation.manifest["datasets"]], [3, 2, 2]
)
self.assertEqual(
[p["part_number"] for p in generation.manifest["parts"]], [1, 2, 1, 1]
)
for part in generation.manifest["parts"]:
for variant in part["variants"]:
self.assertIn(f'-{part["year"]}-part-', variant["file_name"])
variant = next(v for v in part["variants"] if v["format"] == "json")
rows = json.loads(
(_directory(generation) / variant["file_name"]).read_bytes()
)
projected = [r for org in rows for r in org[GROUP]]
self.assertEqual(
{int(r["record_date"][:4]) for r in projected}, {part["year"]}
)
ticket = issue_ticket(
self.user, self.selector(generation, "source_archive", "json")
)
with zipfile.ZipFile(
BytesIO(self.bytes(consume_ticket(ticket["ticket"])))
) as archive:
self.assertEqual(
{n.split("/")[0] for n in archive.namelist() if n != "manifest.json"},
{"2024", "2025", "2026"},
)
self.assertEqual(len(archive.namelist()), 5)
manifest = json.loads(archive.read("manifest.json"))
self.assertEqual(manifest["partitioning"], "year")
self.assertEqual(manifest["records_count"], len(records))
def test_financial_record_without_lines_and_invalid_line_year_are_retained(self):
extension = FinancialIndicatorsExtension.objects.create(
organization=self.organization
)
empty = OrganizationSourceRecord.objects.create(
extension=extension,
source="fns_reports",
record_type="financial_report",
external_id=str(uuid4()),
record_date="2019-01-01",
)
invalid = OrganizationSourceRecord.objects.create(
extension=extension,
source="fns_reports",
record_type="financial_report",
external_id=str(uuid4()),
)
OrganizationSourceFinancialLine.objects.create(
source_record=invalid, year=0, form_code="1", line_code="1"
)
generation = build_source("financial_indicators")
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2019, None]
)
self.assertEqual(generation.manifest["records_count"], 2)
self.assertEqual(generation.manifest["exported_records_count"], 2)
rows = []
for part in generation.manifest["parts"]:
variant = part["variants"][0]
rows.extend(
r
for org in json.loads(
(_directory(generation) / variant["file_name"]).read_bytes()
)
for r in org["financial_indicators"]
)
self.assertEqual({r["uid"] for r in rows}, {str(empty.uid), str(invalid.uid)})
def test_pre_partition_generation_metadata_and_ticket_stay_readable(self):
self.records(1)
generation = build_source(GROUP)
manifest = generation.manifest
manifest["version"] = 3
manifest["description"]["example_dataset_key"] = "records"
manifest.pop("partitioning")
manifest.pop("exported_records_count")
for part in manifest["parts"]:
part.pop("year")
part["dataset_key"] = "records"
for dataset in manifest["datasets"]:
dataset.pop("year")
dataset["key"] = "records"
dataset["export_schema_version"] = "1"
generation.save(update_fields=["manifest"])
ticket = issue_ticket(self.user, self.selector(generation))
expected = (_directory(generation) / ticket["file_name"]).read_bytes()
build_source(GROUP)
data, _ = detail(GROUP, QueryDict(f"generation_id={generation.pk}"))
self.assertEqual(data["partitioning"], "none")
self.assertIsNone(data["files"][0]["year"])
self.assertIsNone(data["datasets"][0]["year"])
self.assertEqual(self.bytes(consume_ticket(ticket["ticket"])), expected)
def test_nightly_dispatch_builds_twenty_two_sources_for_both_years(self):
from organizations.source_groups import SOURCE_GROUP_DESCRIPTORS
from organizations.tasks import refresh_source_export_catalog
for group in registry():
descriptor = next(
d for d in SOURCE_GROUP_DESCRIPTORS.values() if d.source_group == group
)
extension = descriptor.extension_model.objects.filter(
organization=self.organization
).first() or descriptor.extension_model.objects.create(
organization=self.organization
)
for year in [2020, 2021]:
record = OrganizationSourceRecord.objects.create(
extension=extension,
source=descriptor.source,
record_type=descriptor.record_type,
external_id=str(uuid4()),
record_date=f"{year}-01-01",
)
if group == "financial_indicators":
OrganizationSourceFinancialLine.objects.create(
source_record=record, year=year, form_code="1", line_code="1"
)
with patch.object(
refresh_source_export_catalog,
"delay",
side_effect=lambda **kw: refresh_source_export_catalog.run(**kw),
):
result = refresh_source_export_catalog.run()
self.assertEqual(result["groups"], 22)
self.assertEqual(SourceExportGeneration.objects.count(), 22)
for generation in SourceExportGeneration.objects.all():
self.assertEqual(generation.manifest["partitioning"], "year")
self.assertEqual(
[d["year"] for d in generation.manifest["datasets"]], [2020, 2021]
)
self.assertEqual(generation.manifest["records_count"], 2)
def test_nightly_migration_enables_recurring_all_sources_schedule(self):
from importlib import import_module
from django.apps import apps
from django_celery_beat.models import PeriodicTask
migration = import_module(
"organizations.migrations.0021_enable_yearly_partitioned_exports"
)
migration.enable(apps, None)
task = PeriodicTask.objects.get(name=migration.NAME)
self.assertTrue(task.enabled)
self.assertFalse(task.one_off)
self.assertEqual(task.kwargs, "{}")
self.assertEqual(task.crontab.hour, "6")
self.assertEqual(task.crontab.minute, "30")
self.assertEqual(str(task.crontab.timezone), "Europe/Moscow")
task.one_off = True
task.expires = timezone.now() - timedelta(days=1)
task.start_time = timezone.now() + timedelta(days=10)
task.kwargs = '{"source_group":"arbitration"}'
task.save()
migration.enable(apps, None)
task.refresh_from_db()
self.assertFalse(task.one_off)
self.assertIsNone(task.expires)
self.assertIsNone(task.expire_seconds)
self.assertIsNone(task.start_time)
self.assertEqual(task.kwargs, "{}")
self.assertEqual(PeriodicTask.objects.filter(name=migration.NAME).count(), 1)
def test_nightly_migration_notifies_beat_without_live_model_signals(self):
from importlib import import_module
from django.apps import apps
from django_celery_beat.models import PeriodicTasks
migration = import_module(
"organizations.migrations.0021_enable_yearly_partitioned_exports"
)
# Historical migration models do not execute the live save hooks.
with patch.object(PeriodicTasks, "update_changed"):
PeriodicTasks.objects.all().delete()
migration.enable(apps, None)
self.assertTrue(PeriodicTasks.objects.filter(ident=1).exists())
old = timezone.now() - timedelta(days=30)
PeriodicTasks.objects.filter(ident=1).update(last_update=old)
migration.disable(apps, None)
self.assertGreater(PeriodicTasks.objects.get(ident=1).last_update, old)