feat: limit source exports to current year
All checks were successful
All checks were successful
This commit is contained in:
@@ -9,6 +9,11 @@
|
||||
теле обычной HTML-формы на
|
||||
`POST /api/v2/organization-source-records/export-download/`.
|
||||
|
||||
Имя скачиваемого архива формируется по выбранным источникам и времени создания
|
||||
запроса: `<source-stem>_YYYYMMDD_HHMMSS.zip`, а для нескольких источников их
|
||||
имена соединяются через `__`. Значение `file_name` из ticket и итоговый
|
||||
`Content-Disposition` совпадают.
|
||||
|
||||
Браузер получает потоковый ZIP напрямую, без многогигабайтного `Blob` в
|
||||
JavaScript. Ticket не попадает в URL и после первого запроса становится
|
||||
недействительным. Совместимый администраторский endpoint
|
||||
@@ -19,6 +24,13 @@ JavaScript. Ticket не попадает в URL и после первого з
|
||||
готовые файлы последнего ночного поколения. При отсутствии поколения API
|
||||
возвращает `503` с кодом `source_export_not_ready`.
|
||||
|
||||
Каждое поколение содержит только текущий календарный год в timezone сервиса.
|
||||
Для моделей с предметной датой год определяется по ней, для моделей без такой
|
||||
даты — по `created_at`, для финансовых отчётов — по году строк отчёта. Строки
|
||||
других лет из финансового отчёта не выгружаются. После смены года старое
|
||||
поколение не раздаётся: до успешной сборки нового года API отвечает
|
||||
`503 source_export_not_ready`.
|
||||
|
||||
## Матрица
|
||||
|
||||
| Группа API | Таблицы State Corp | Файл | CSV | XLSX | JSON |
|
||||
@@ -39,9 +51,9 @@ JavaScript. Ticket не попадает в URL и после первого з
|
||||
используют тот же контракт, что и Mostovik: реквизиты организации, включая ОКПО,
|
||||
общие поля записи источника и специфичные поля в `payload.*`.
|
||||
|
||||
Все записи включаются в публичные файлы, а техническое наименование внешнего
|
||||
поставщика нейтрализуется. Исходные значения в БД сохраняются для работы
|
||||
интеграции и дедупликации.
|
||||
Все записи текущего года включаются в публичные файлы, а техническое
|
||||
наименование внешнего поставщика нейтрализуется. Исходные значения в БД
|
||||
сохраняются для работы интеграции и дедупликации.
|
||||
|
||||
Физических XLSX-файлов может быть больше: по умолчанию один файл содержит не
|
||||
более 100 000 строк данных и получает суффикс `-part-001`, `-part-002` и далее.
|
||||
@@ -55,11 +67,13 @@ Celery Beat запускает
|
||||
Генератор:
|
||||
|
||||
1. читает каждую нормализованную таблицу один раз без model-level сортировки;
|
||||
предметные даты и fallback по `created_at` индексированы;
|
||||
2. создаёт компактный JSON-массив и переиспользует его как готовый JSON;
|
||||
3. потоково формирует CSV и write-only XLSX;
|
||||
4. атомарно публикует `current.json` только после готовности всей матрицы;
|
||||
5. при ошибке удаляет staging и продолжает отдавать предыдущее поколение;
|
||||
6. сохраняет текущее и предыдущее поколения по умолчанию.
|
||||
4. записывает календарный `export_year` в manifest;
|
||||
5. атомарно публикует `current.json` только после готовности всей матрицы;
|
||||
6. при ошибке удаляет staging и продолжает отдавать предыдущее поколение того же года;
|
||||
7. сохраняет текущее и предыдущее поколения по умолчанию.
|
||||
|
||||
Web и Celery worker должны использовать общий read-write volume `/app/media`.
|
||||
|
||||
@@ -81,3 +95,8 @@ PYTHONPATH=src uv run python src/manage.py build_source_record_exports
|
||||
|
||||
Для атомарной генерации требуется свободное место не меньше
|
||||
`(GENERATIONS_TO_KEEP + 1) * размер поколения` плюс запас файловой системы.
|
||||
|
||||
На локальном снимке от 2026-08-04 поколение за 2026 год содержит 1 962 записи
|
||||
и занимает 4 738 872 байта (4,52 MiB) для всей матрицы. Архив со всеми
|
||||
источниками оценивается в 2,49 MiB для JSON, 1,59 MiB для CSV и 0,44 MiB для
|
||||
XLSX; к сумме добавляется небольшой служебный overhead ZIP.
|
||||
|
||||
@@ -20,6 +20,7 @@ class Command(BaseAppCommand):
|
||||
{
|
||||
"generation_id": generation.generation_id,
|
||||
"generated_at": generation.generated_at,
|
||||
"export_year": generation.export_year,
|
||||
"artifacts_count": generation.artifacts_count,
|
||||
"files_count": generation.files_count,
|
||||
"records_count": generation.records_count,
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Index the remaining date fields used by current-year exports."""
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
"""Add indexes for nullable certificate and security-registry dates."""
|
||||
|
||||
dependencies = [
|
||||
("external_data", "0007_seed_nightly_source_record_exports"),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name="industrialcertificate",
|
||||
name="issue_date",
|
||||
field=models.DateField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
null=True,
|
||||
verbose_name="дата выдачи",
|
||||
),
|
||||
),
|
||||
migrations.AlterField(
|
||||
model_name="informationsecurityregistryentry",
|
||||
name="issued_at",
|
||||
field=models.DateField(
|
||||
blank=True,
|
||||
db_index=True,
|
||||
null=True,
|
||||
verbose_name="дата выдачи",
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -44,7 +44,12 @@ class IndustrialCertificate(UUIDPrimaryKeyMixin, TimestampMixin, models.Model):
|
||||
certificate_number = models.CharField(
|
||||
_("номер сертификата"), max_length=100, db_index=True
|
||||
)
|
||||
issue_date = models.DateField(_("дата выдачи"), null=True, blank=True)
|
||||
issue_date = models.DateField(
|
||||
_("дата выдачи"),
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
expiry_date = models.DateField(_("дата окончания"), null=True, blank=True)
|
||||
certificate_file_url = models.TextField(
|
||||
_("ссылка на файл сертификата"), blank=True, default=""
|
||||
@@ -254,7 +259,12 @@ class InformationSecurityRegistryEntry(
|
||||
blank=True,
|
||||
default="",
|
||||
)
|
||||
issued_at = models.DateField(_("дата выдачи"), null=True, blank=True)
|
||||
issued_at = models.DateField(
|
||||
_("дата выдачи"),
|
||||
null=True,
|
||||
blank=True,
|
||||
db_index=True,
|
||||
)
|
||||
expires_at = models.DateField(_("дата окончания"), null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
|
||||
@@ -24,6 +24,7 @@ from apps.external_data.models import (
|
||||
BankruptcyProcedure,
|
||||
DefenseUnreliableSupplier,
|
||||
FinancialReport,
|
||||
FinancialReportLine,
|
||||
IndustrialCertificate,
|
||||
IndustrialProduct,
|
||||
InformationSecurityRegistryEntry,
|
||||
@@ -34,7 +35,7 @@ from apps.external_data.models import (
|
||||
)
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db.models import QuerySet, prefetch_related_objects
|
||||
from django.db.models import Prefetch, Q, QuerySet, prefetch_related_objects
|
||||
from django.utils import timezone
|
||||
from openpyxl import Workbook
|
||||
|
||||
@@ -43,7 +44,7 @@ EXPORT_FORMAT_XLSX = "xlsx"
|
||||
EXPORT_FORMAT_JSON = "json"
|
||||
EXPORT_FORMATS = (EXPORT_FORMAT_CSV, EXPORT_FORMAT_XLSX, EXPORT_FORMAT_JSON)
|
||||
FINANCIAL_SOURCE_GROUP = "financial_indicators"
|
||||
EXPORT_MANIFEST_VERSION = 1
|
||||
EXPORT_MANIFEST_VERSION = 2
|
||||
CURRENT_EXPORT_MANIFEST_FILE_NAME = "current.json"
|
||||
GENERATION_MANIFEST_FILE_NAME = "manifest.json"
|
||||
GENERATION_DIRECTORY_NAME = "generations"
|
||||
@@ -117,6 +118,7 @@ class SourceModelExportSpec:
|
||||
payload_aliases: tuple[tuple[str, str], ...] = ()
|
||||
payload_export_fields: tuple[str, ...] | None = None
|
||||
prefetch_related: tuple[str, ...] = ()
|
||||
export_year_lookup: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -153,6 +155,7 @@ SOURCE_GROUP_EXPORT_SPECS: dict[str, SourceGroupExportSpec] = {
|
||||
status_field="status",
|
||||
load_batch_field="load_batch",
|
||||
prefetch_related=("lines",),
|
||||
export_year_lookup="lines__year",
|
||||
),
|
||||
),
|
||||
),
|
||||
@@ -436,6 +439,7 @@ class SourceRecordExportGeneration:
|
||||
|
||||
generation_id: str
|
||||
generated_at: str
|
||||
export_year: int
|
||||
artifacts: tuple[SourceRecordExportArtifact, ...]
|
||||
records_count: int
|
||||
|
||||
@@ -514,6 +518,7 @@ def build_source_record_export_artifacts(
|
||||
generations_directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
|
||||
export_year = _export_year(generated_at_datetime)
|
||||
generation_id = (
|
||||
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
|
||||
)
|
||||
@@ -530,6 +535,7 @@ def build_source_record_export_artifacts(
|
||||
headers, records_count = _spool_source_group_rows(
|
||||
source_spec=source_spec,
|
||||
output_path=row_spool_path,
|
||||
export_year=export_year,
|
||||
)
|
||||
source_record_counts[source_group] = records_count
|
||||
|
||||
@@ -569,6 +575,7 @@ def build_source_record_export_artifacts(
|
||||
generation = SourceRecordExportGeneration(
|
||||
generation_id=generation_id,
|
||||
generated_at=generated_at_datetime.isoformat(),
|
||||
export_year=export_year,
|
||||
artifacts=tuple(artifacts),
|
||||
records_count=sum(source_record_counts.values()),
|
||||
)
|
||||
@@ -622,6 +629,7 @@ def build_source_records_export_archive(
|
||||
source_groups: Sequence[str],
|
||||
export_format: str,
|
||||
export_directory: str | Path | None = None,
|
||||
requested_at: datetime | None = None,
|
||||
) -> SourceRecordExportArchive:
|
||||
"""Package selected prepared files without querying external-data tables."""
|
||||
|
||||
@@ -629,6 +637,13 @@ def build_source_records_export_archive(
|
||||
generation = load_current_source_record_export_generation(
|
||||
export_directory=root_directory,
|
||||
)
|
||||
requested_at_datetime = _normalize_generation_datetime(
|
||||
requested_at or timezone.now()
|
||||
)
|
||||
if generation.export_year != _export_year(requested_at_datetime):
|
||||
raise SourceRecordExportArtifactsUnavailable(
|
||||
"Prepared source-record export belongs to a different calendar year."
|
||||
)
|
||||
artifacts_by_key: dict[
|
||||
tuple[str, str],
|
||||
list[SourceRecordExportArtifact],
|
||||
@@ -654,10 +669,11 @@ def build_source_records_export_archive(
|
||||
sorted(artifacts, key=lambda artifact: artifact.part_number)
|
||||
)
|
||||
|
||||
generated_at = datetime.fromisoformat(generation.generated_at)
|
||||
timestamp = generated_at.strftime("%Y%m%d_%H%M%S")
|
||||
return SourceRecordExportArchive(
|
||||
archive_name=f"organization_source_records_export_{timestamp}.zip",
|
||||
archive_name=_build_source_records_archive_name(
|
||||
source_groups=source_groups,
|
||||
requested_at=requested_at_datetime,
|
||||
),
|
||||
archive_chunks=_stream_zip_archive(selected_artifacts),
|
||||
files_count=len(selected_artifacts),
|
||||
generated_at=generation.generated_at,
|
||||
@@ -671,9 +687,11 @@ def create_source_record_export_download_ticket(
|
||||
) -> SourceRecordExportDownloadTicket:
|
||||
"""Validate prepared files and cache a short-lived download capability."""
|
||||
|
||||
requested_at = _normalize_generation_datetime(timezone.now())
|
||||
package = build_source_records_export_archive(
|
||||
source_groups=source_groups,
|
||||
export_format=export_format,
|
||||
requested_at=requested_at,
|
||||
)
|
||||
expires_in = max(
|
||||
1,
|
||||
@@ -685,7 +703,11 @@ def create_source_record_export_download_ticket(
|
||||
)
|
||||
),
|
||||
)
|
||||
payload = {"sources": list(source_groups), "format": export_format}
|
||||
payload = {
|
||||
"sources": list(source_groups),
|
||||
"format": export_format,
|
||||
"requested_at": requested_at.isoformat(),
|
||||
}
|
||||
for _attempt in range(3):
|
||||
ticket = secrets.token_urlsafe(32)
|
||||
if cache.add(
|
||||
@@ -718,6 +740,7 @@ def consume_source_record_export_download_ticket(
|
||||
try:
|
||||
source_groups = payload["sources"]
|
||||
export_format = payload["format"]
|
||||
requested_at_value = payload["requested_at"]
|
||||
if (
|
||||
not isinstance(source_groups, list)
|
||||
or not source_groups
|
||||
@@ -728,14 +751,19 @@ def consume_source_record_export_download_ticket(
|
||||
)
|
||||
or len(source_groups) != len(set(source_groups))
|
||||
or export_format not in EXPORT_FORMATS
|
||||
or not isinstance(requested_at_value, str)
|
||||
):
|
||||
raise ValueError
|
||||
requested_at = datetime.fromisoformat(requested_at_value)
|
||||
if timezone.is_naive(requested_at):
|
||||
raise ValueError
|
||||
except (KeyError, TypeError, ValueError):
|
||||
raise SourceRecordExportTicketInvalid from None
|
||||
|
||||
return build_source_records_export_archive(
|
||||
source_groups=source_groups,
|
||||
export_format=export_format,
|
||||
requested_at=requested_at,
|
||||
)
|
||||
|
||||
|
||||
@@ -788,6 +816,24 @@ def _normalize_generation_datetime(value: datetime) -> datetime:
|
||||
return value.astimezone(UTC)
|
||||
|
||||
|
||||
def _export_year(value: datetime) -> int:
|
||||
"""Return the calendar year in the configured service timezone."""
|
||||
return timezone.localtime(value).year
|
||||
|
||||
|
||||
def _build_source_records_archive_name(
|
||||
*,
|
||||
source_groups: Sequence[str],
|
||||
requested_at: datetime,
|
||||
) -> str:
|
||||
source_name = "__".join(
|
||||
SOURCE_GROUP_EXPORT_SPECS[source_group].file_stem
|
||||
for source_group in source_groups
|
||||
)
|
||||
timestamp = timezone.localtime(requested_at).strftime("%Y%m%d_%H%M%S")
|
||||
return f"{source_name}_{timestamp}.zip"
|
||||
|
||||
|
||||
def _source_group_export_formats(source_group: str) -> tuple[str, ...]:
|
||||
if source_group == FINANCIAL_SOURCE_GROUP:
|
||||
return (EXPORT_FORMAT_JSON,)
|
||||
@@ -808,25 +854,59 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str
|
||||
return f"{SOURCE_GROUP_EXPORT_SPECS[source_group].file_stem}.{file_format}"
|
||||
|
||||
|
||||
def _source_model_queryset(model_spec: SourceModelExportSpec) -> QuerySet:
|
||||
return cast(
|
||||
def _source_model_queryset(
|
||||
model_spec: SourceModelExportSpec,
|
||||
*,
|
||||
export_year: int,
|
||||
) -> QuerySet:
|
||||
queryset = cast(
|
||||
QuerySet,
|
||||
model_spec.model.objects.select_related("organization").order_by(),
|
||||
)
|
||||
if model_spec.export_year_lookup:
|
||||
return queryset.filter(
|
||||
**{model_spec.export_year_lookup: export_year}
|
||||
).distinct()
|
||||
if model_spec.record_date_field:
|
||||
return queryset.filter(
|
||||
Q(**{f"{model_spec.record_date_field}__year": export_year})
|
||||
| Q(
|
||||
**{
|
||||
f"{model_spec.record_date_field}__isnull": True,
|
||||
"created_at__year": export_year,
|
||||
}
|
||||
)
|
||||
)
|
||||
return queryset.filter(created_at__year=export_year)
|
||||
|
||||
|
||||
def _iter_source_model_records(
|
||||
model_spec: SourceModelExportSpec,
|
||||
*,
|
||||
export_year: int,
|
||||
) -> Iterator[Any]:
|
||||
iterator = _source_model_queryset(model_spec).iterator(
|
||||
chunk_size=SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE
|
||||
)
|
||||
iterator = _source_model_queryset(
|
||||
model_spec,
|
||||
export_year=export_year,
|
||||
).iterator(chunk_size=SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE)
|
||||
while True:
|
||||
batch = list(islice(iterator, SOURCE_RECORD_EXPORT_ITERATOR_CHUNK_SIZE))
|
||||
if not batch:
|
||||
return
|
||||
if model_spec.prefetch_related:
|
||||
prefetch_related_objects(batch, *model_spec.prefetch_related)
|
||||
prefetches = [
|
||||
Prefetch(
|
||||
related_name,
|
||||
queryset=FinancialReportLine.objects.filter(
|
||||
year=export_year
|
||||
).order_by(),
|
||||
)
|
||||
if related_name == "lines"
|
||||
and model_spec.export_year_lookup == "lines__year"
|
||||
else related_name
|
||||
for related_name in model_spec.prefetch_related
|
||||
]
|
||||
prefetch_related_objects(batch, *prefetches)
|
||||
yield from batch
|
||||
|
||||
|
||||
@@ -851,13 +931,17 @@ def _spool_source_group_rows(
|
||||
*,
|
||||
source_spec: SourceGroupExportSpec,
|
||||
output_path: Path,
|
||||
export_year: int,
|
||||
) -> tuple[list[str], int]:
|
||||
records_count = 0
|
||||
with output_path.open("w", encoding="utf-8", newline="") as output:
|
||||
output.write("[")
|
||||
is_first_row = True
|
||||
for model_spec in source_spec.models:
|
||||
for record in _iter_source_model_records(model_spec):
|
||||
for record in _iter_source_model_records(
|
||||
model_spec,
|
||||
export_year=export_year,
|
||||
):
|
||||
row = _build_record_row(
|
||||
record,
|
||||
source_spec=source_spec,
|
||||
@@ -1149,6 +1233,7 @@ def _generation_manifest_payload(
|
||||
"version": EXPORT_MANIFEST_VERSION,
|
||||
"generation_id": generation.generation_id,
|
||||
"generated_at": generation.generated_at,
|
||||
"export_year": generation.export_year,
|
||||
"records_count": generation.records_count,
|
||||
"artifacts_count": generation.artifacts_count,
|
||||
"files_count": generation.files_count,
|
||||
@@ -1180,6 +1265,9 @@ def _generation_from_manifest(
|
||||
generation_id = str(payload["generation_id"])
|
||||
generated_at = str(payload["generated_at"])
|
||||
datetime.fromisoformat(generated_at)
|
||||
export_year = int(payload["export_year"])
|
||||
if not 1 <= export_year <= 9999:
|
||||
raise ValueError("Source-record export year is invalid.")
|
||||
records_count = int(payload["records_count"])
|
||||
artifact_payloads = payload["artifacts"]
|
||||
if not isinstance(artifact_payloads, list):
|
||||
@@ -1234,6 +1322,7 @@ def _generation_from_manifest(
|
||||
return SourceRecordExportGeneration(
|
||||
generation_id=generation_id,
|
||||
generated_at=generated_at,
|
||||
export_year=export_year,
|
||||
artifacts=tuple(artifacts),
|
||||
records_count=records_count,
|
||||
)
|
||||
|
||||
@@ -35,6 +35,7 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
||||
"status": "success",
|
||||
"generation_id": generation.generation_id,
|
||||
"generated_at": generation.generated_at,
|
||||
"export_year": generation.export_year,
|
||||
"artifacts_count": generation.artifacts_count,
|
||||
"files_count": generation.files_count,
|
||||
"records_count": generation.records_count,
|
||||
|
||||
@@ -8,6 +8,7 @@ from django.apps import apps as django_apps
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.test import TestCase, override_settings
|
||||
from django.utils import timezone
|
||||
from django_celery_beat.models import PeriodicTask
|
||||
|
||||
|
||||
@@ -36,6 +37,7 @@ class SourceRecordExportArtifactsTaskTest(TestCase):
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["artifacts_count"], 25)
|
||||
self.assertEqual(result["export_year"], timezone.localdate().year)
|
||||
self.assertIsNone(cache.get(settings.SOURCE_RECORD_EXPORT_LOCK_KEY))
|
||||
|
||||
def test_refresh_task_skips_when_generation_lock_is_held(self):
|
||||
|
||||
@@ -2,19 +2,24 @@
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from datetime import UTC, datetime
|
||||
from io import BytesIO, StringIO
|
||||
from tempfile import TemporaryDirectory
|
||||
from unittest.mock import patch
|
||||
|
||||
from apps.external_data.source_record_export import (
|
||||
ORGANIZATION_EXPORT_FIELDS,
|
||||
SOURCE_GROUP_EXPORT_SPECS,
|
||||
SOURCE_RECORD_EXPORT_FIELDS,
|
||||
SourceRecordExportArtifactsUnavailable,
|
||||
_source_group_headers,
|
||||
build_source_record_export_artifacts,
|
||||
build_source_records_export_archive,
|
||||
load_current_source_record_export_generation,
|
||||
)
|
||||
from django.core.management import call_command
|
||||
from django.test import override_settings
|
||||
from django.utils import timezone
|
||||
from openpyxl import load_workbook
|
||||
from rest_framework import status
|
||||
from rest_framework.test import APITestCase
|
||||
@@ -28,6 +33,7 @@ from tests.apps.external_data.factories import (
|
||||
IndustrialProductFactory,
|
||||
ManufacturerRegistryEntryFactory,
|
||||
ProsecutorCheckFactory,
|
||||
PublicProcurementFactory,
|
||||
)
|
||||
from tests.apps.organization.factories import OrganizationFactory
|
||||
from tests.apps.user.factories import UserFactory
|
||||
@@ -71,23 +77,38 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
self.assertEqual(response["Retry-After"], "3600")
|
||||
|
||||
def test_generation_builds_full_matrix_from_normalized_tables(self):
|
||||
current_date = timezone.localdate()
|
||||
organization = OrganizationFactory.create(
|
||||
full_name='Акционерное общество "Экспорт"',
|
||||
okpo="12345678",
|
||||
)
|
||||
IndustrialProductFactory.create(organization=organization)
|
||||
IndustrialCertificateFactory.create(organization=organization)
|
||||
IndustrialCertificateFactory.create(
|
||||
organization=organization,
|
||||
issue_date=current_date,
|
||||
)
|
||||
ManufacturerRegistryEntryFactory.create(organization=organization)
|
||||
ProsecutorCheckFactory.create(organization=organization)
|
||||
arbitration_case = ArbitrationCaseFactory.create(organization=organization)
|
||||
ProsecutorCheckFactory.create(
|
||||
organization=organization,
|
||||
start_date=current_date,
|
||||
)
|
||||
arbitration_case = ArbitrationCaseFactory.create(
|
||||
organization=organization,
|
||||
decision_date=current_date,
|
||||
)
|
||||
report = FinancialReportFactory.create(organization=organization)
|
||||
FinancialReportLineFactory.create(report=report, line_code="1600")
|
||||
FinancialReportLineFactory.create(
|
||||
report=report,
|
||||
line_code="1600",
|
||||
year=current_date.year,
|
||||
)
|
||||
|
||||
generation = build_source_record_export_artifacts()
|
||||
|
||||
self.assertEqual(generation.artifacts_count, 25)
|
||||
self.assertEqual(generation.files_count, 25)
|
||||
self.assertEqual(generation.records_count, 6)
|
||||
self.assertEqual(generation.export_year, current_date.year)
|
||||
expected_prefix = [*ORGANIZATION_EXPORT_FIELDS, *SOURCE_RECORD_EXPORT_FIELDS]
|
||||
for source_spec in SOURCE_GROUP_EXPORT_SPECS.values():
|
||||
self.assertEqual(
|
||||
@@ -229,14 +250,104 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
|
||||
generation = load_current_source_record_export_generation()
|
||||
self.assertEqual(generation.artifacts_count, 25)
|
||||
self.assertEqual(generation.export_year, timezone.localdate().year)
|
||||
self.assertIn('"artifacts_count": 25', command_output.getvalue())
|
||||
|
||||
def test_generation_contains_only_records_from_its_calendar_year(self):
|
||||
export_year = 2026
|
||||
generated_at = datetime(export_year, 8, 4, 6, 0, tzinfo=UTC)
|
||||
organization = OrganizationFactory.create()
|
||||
current_procurement = PublicProcurementFactory.create(
|
||||
organization=organization,
|
||||
contract_date=datetime(2026, 3, 1, tzinfo=UTC).date(),
|
||||
)
|
||||
PublicProcurementFactory.create(
|
||||
organization=organization,
|
||||
contract_date=datetime(2025, 3, 1, tzinfo=UTC).date(),
|
||||
)
|
||||
current_product = IndustrialProductFactory.create(organization=organization)
|
||||
old_product = IndustrialProductFactory.create(organization=organization)
|
||||
current_product.__class__.objects.filter(pk=current_product.pk).update(
|
||||
created_at=generated_at
|
||||
)
|
||||
old_product.__class__.objects.filter(pk=old_product.pk).update(
|
||||
created_at=datetime(2025, 8, 4, 6, 0, tzinfo=UTC)
|
||||
)
|
||||
|
||||
current_report = FinancialReportFactory.create(organization=organization)
|
||||
old_report = FinancialReportFactory.create(organization=organization)
|
||||
FinancialReportLineFactory.create(report=current_report, year=2025)
|
||||
FinancialReportLineFactory.create(
|
||||
report=current_report,
|
||||
year=export_year,
|
||||
line_code="1601",
|
||||
)
|
||||
FinancialReportLineFactory.create(report=old_report, year=2025)
|
||||
|
||||
generation = build_source_record_export_artifacts(now=generated_at)
|
||||
|
||||
self.assertEqual(generation.export_year, export_year)
|
||||
procurements_path = next(
|
||||
artifact.path
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == "government_procurements"
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
procurement_rows = json.loads(procurements_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in procurement_rows],
|
||||
[str(current_procurement.pk)],
|
||||
)
|
||||
|
||||
industrial_path = next(
|
||||
artifact.path
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == "industrial_production"
|
||||
and artifact.file_format == "json"
|
||||
)
|
||||
industrial_rows = json.loads(industrial_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in industrial_rows],
|
||||
[str(current_product.pk)],
|
||||
)
|
||||
|
||||
financial_path = next(
|
||||
artifact.path
|
||||
for artifact in generation.artifacts
|
||||
if artifact.source_group == "financial_indicators"
|
||||
)
|
||||
financial_rows = json.loads(financial_path.read_text(encoding="utf-8"))
|
||||
self.assertEqual(
|
||||
[row["uid"] for row in financial_rows],
|
||||
[str(current_report.pk)],
|
||||
)
|
||||
self.assertEqual(
|
||||
{line["year"] for line in financial_rows[0]["financial_lines"]},
|
||||
{export_year},
|
||||
)
|
||||
|
||||
def test_new_calendar_year_requires_a_new_prepared_generation(self):
|
||||
build_source_record_export_artifacts(
|
||||
now=datetime(2026, 12, 31, 23, 59, tzinfo=UTC)
|
||||
)
|
||||
|
||||
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
|
||||
build_source_records_export_archive(
|
||||
source_groups=["planned_inspections"],
|
||||
export_format="json",
|
||||
requested_at=datetime(2027, 1, 1, tzinfo=UTC),
|
||||
)
|
||||
|
||||
def test_admin_streams_selected_prepared_files_without_database_queries(self):
|
||||
self.client.force_authenticate(UserFactory.create_superuser())
|
||||
organization = OrganizationFactory.create(okpo="87654321")
|
||||
ProsecutorCheckFactory.create(organization=organization)
|
||||
current_date = timezone.localdate()
|
||||
ProsecutorCheckFactory.create(
|
||||
organization=organization,
|
||||
start_date=current_date,
|
||||
)
|
||||
report = FinancialReportFactory.create(organization=organization)
|
||||
FinancialReportLineFactory.create(report=report)
|
||||
FinancialReportLineFactory.create(report=report, year=current_date.year)
|
||||
generation = build_source_record_export_artifacts()
|
||||
|
||||
with self.assertNumQueries(0):
|
||||
@@ -274,7 +385,8 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
self.assertEqual(rows[1][4], "87654321")
|
||||
|
||||
def test_ticket_is_admin_only_and_can_be_consumed_once_without_auth(self):
|
||||
build_source_record_export_artifacts()
|
||||
requested_at = datetime(2026, 8, 4, 12, 34, 56, tzinfo=UTC)
|
||||
build_source_record_export_artifacts(now=requested_at)
|
||||
regular_user = UserFactory.create_user()
|
||||
self.client.force_authenticate(regular_user)
|
||||
forbidden_response = self.client.post(
|
||||
@@ -285,7 +397,13 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
self.assertEqual(forbidden_response.status_code, status.HTTP_403_FORBIDDEN)
|
||||
|
||||
self.client.force_authenticate(UserFactory.create_superuser())
|
||||
with self.assertNumQueries(0):
|
||||
with (
|
||||
patch(
|
||||
"apps.external_data.source_record_export.timezone.now",
|
||||
return_value=requested_at,
|
||||
),
|
||||
self.assertNumQueries(0),
|
||||
):
|
||||
ticket_response = self.client.post(
|
||||
self.ticket_url,
|
||||
{"sources": ["bankruptcy"], "format": "json"},
|
||||
@@ -294,6 +412,10 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
self.assertEqual(ticket_response.status_code, status.HTTP_201_CREATED)
|
||||
self.assertRegex(ticket_response.data["ticket"], r"^[A-Za-z0-9_-]{43}$")
|
||||
self.assertEqual(ticket_response.data["expires_in"], 300)
|
||||
self.assertEqual(
|
||||
ticket_response.data["file_name"],
|
||||
"bankruptcy-procedures_20260804_123456.zip",
|
||||
)
|
||||
|
||||
self.client.force_authenticate(user=None)
|
||||
with self.assertNumQueries(0):
|
||||
@@ -303,6 +425,10 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
format="multipart",
|
||||
)
|
||||
self.assertEqual(download_response.status_code, status.HTTP_200_OK)
|
||||
self.assertEqual(
|
||||
download_response["Content-Disposition"],
|
||||
'attachment; filename="bankruptcy-procedures_20260804_123456.zip"',
|
||||
)
|
||||
with zipfile.ZipFile(
|
||||
BytesIO(self._response_body(download_response))
|
||||
) as archive:
|
||||
@@ -319,7 +445,11 @@ class SourceRecordExportApiTest(APITestCase):
|
||||
@override_settings(SOURCE_RECORD_EXPORT_XLSX_ROWS_PER_FILE=2)
|
||||
def test_xlsx_is_split_into_bounded_workbook_parts(self):
|
||||
organization = OrganizationFactory.create()
|
||||
ProsecutorCheckFactory.create_batch(3, organization=organization)
|
||||
ProsecutorCheckFactory.create_batch(
|
||||
3,
|
||||
organization=organization,
|
||||
start_date=timezone.localdate(),
|
||||
)
|
||||
|
||||
generation = build_source_record_export_artifacts()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user