feat: add separate all-years source export task

This commit is contained in:
Aleksandr Meshchryakov
2026-09-28 12:06:09 +02:00
parent 245edd1fff
commit cd5a1764a8
5 changed files with 187 additions and 13 deletions

View File

@@ -15,8 +15,18 @@ class Command(BaseAppCommand):
help = "Формирует готовые CSV/XLSX/JSON выгрузки источников"
use_transaction = False
def add_arguments(self, parser) -> None:
super().add_arguments(parser)
parser.add_argument(
"--all-years",
action="store_true",
help="Все доступные годы в отдельном каталоге all-years",
)
def execute_command(self, *args, **options) -> str:
generation = build_source_record_export_artifacts()
generation = build_source_record_export_artifacts(
all_years=options["all_years"]
)
rendered = json.dumps(
{
"generation_id": generation.generation_id,

View File

@@ -195,7 +195,7 @@ class SourceRecordExportGeneration:
generation_id: str
generated_at: str
export_year: int
export_year: int | None
artifacts: tuple[SourceRecordExportArtifact, ...]
records_count: int
source_snapshots: dict[str, str] = field(default_factory=dict)
@@ -267,16 +267,19 @@ def build_source_record_export_artifacts(
*,
now: datetime | None = None,
export_directory: str | Path | None = None,
all_years: bool = False,
) -> SourceRecordExportGeneration:
"""Build all files on disk and atomically publish a new generation."""
"""Build annual or all-years files and atomically publish their generation."""
root_directory = _resolve_export_directory(export_directory)
if all_years:
root_directory /= "all-years"
generations_directory = root_directory / GENERATION_DIRECTORY_NAME
root_directory.mkdir(parents=True, exist_ok=True)
generations_directory.mkdir(parents=True, exist_ok=True)
generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
export_year = _export_year(generated_at_datetime)
export_year = None if all_years else _export_year(generated_at_datetime)
generation_id = (
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
)
@@ -739,7 +742,7 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str
def _source_group_queryset(
source_group: str,
*,
export_year: int,
export_year: int | None,
) -> QuerySet[OrganizationSourceRecord]:
queryset = (
OrganizationSourceRecord.objects.filter(extension__source_group=source_group)
@@ -749,6 +752,8 @@ def _source_group_queryset(
# source groups with millions of rows.
.order_by()
)
if export_year is None:
return queryset
if source_group == FINANCIAL_SOURCE_GROUP:
return queryset.filter(financial_lines__year=export_year).distinct()
if source_group in ALL_HISTORY_SOURCE_GROUPS:
@@ -777,7 +782,7 @@ def _iter_source_records(
*,
source_group: str,
include_financial_lines: bool,
export_year: int,
export_year: int | None,
) -> Iterator[OrganizationSourceRecord]:
iterator = _source_group_queryset(
source_group,
@@ -793,7 +798,7 @@ def _iter_source_records(
Prefetch(
"financial_lines",
queryset=OrganizationSourceFinancialLine.objects.filter(
year=export_year
**({"year": export_year} if export_year is not None else {})
).order_by(),
),
)
@@ -804,7 +809,7 @@ def _spool_source_group_rows(
*,
source_group: str,
output_path: Path,
export_year: int,
export_year: int | None,
) -> tuple[list[str], int]:
include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP
payload_headers: set[str] = set()
@@ -1373,6 +1378,9 @@ def _generation_manifest_payload(
"source_snapshots": generation.source_snapshots,
"generated_at": generation.generated_at,
"export_year": generation.export_year,
"export_scope": "all_years"
if generation.export_year is None
else "current_year",
"records_count": generation.records_count,
"artifacts_count": generation.artifacts_count,
"files_count": generation.files_count,
@@ -1403,6 +1411,19 @@ def _manifest_snapshot_ids(payload: dict) -> dict[str, str]:
return identities
def _manifest_export_year(payload: dict[str, Any]) -> int | None:
raw_year = payload["export_year"]
scope = payload.get("export_scope", "current_year")
if scope == "all_years" and raw_year is None:
return None
if scope != "current_year" or raw_year is None or isinstance(raw_year, bool):
raise ValueError("Source-record export scope is invalid.")
year = int(raw_year)
if not 1 <= year <= 9999:
raise ValueError("Source-record export year is invalid.")
return year
def _generation_from_manifest(
payload: Any,
*,
@@ -1415,9 +1436,7 @@ def _generation_from_manifest(
source_snapshots = _manifest_snapshot_ids(payload)
generated_at = str(payload["generated_at"])
datetime.fromisoformat(generated_at)
export_year = int(payload["export_year"])
if not 1 <= export_year <= 9999:
raise ValueError("Source-record export year is invalid.")
export_year = _manifest_export_year(payload)
records_count = int(payload["records_count"])
artifact_payloads = payload["artifacts"]
if not isinstance(artifact_payloads, list):

View File

@@ -22,6 +22,16 @@ logger = logging.getLogger(__name__)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
"""Build and atomically publish the nightly source-record export matrix."""
return _refresh_source_record_export_artifacts(self, all_years=False)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_all_years_source_record_export_artifacts(self) -> dict:
"""Publish all available years independently of the annual generation."""
return _refresh_source_record_export_artifacts(self, all_years=True)
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
lock_key = getattr(
settings,
"SOURCE_RECORD_EXPORT_LOCK_KEY",
@@ -37,7 +47,7 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
countdown = 30
max_retries = max(1, math.ceil(lock_ttl / countdown) + 2)
try:
raise self.retry(countdown=countdown, max_retries=max_retries)
raise task.retry(countdown=countdown, max_retries=max_retries)
except MaxRetriesExceededError:
logger.error(
"Source-record export retry budget exhausted while waiting for build lock"
@@ -45,7 +55,7 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
raise
try:
generation = build_source_record_export_artifacts()
generation = build_source_record_export_artifacts(all_years=all_years)
result = {
"status": "success",
"generation_id": generation.generation_id,