feat: add separate all-years source export task
This commit is contained in:
@@ -14,8 +14,18 @@ class Command(BaseAppCommand):
|
||||
help = "Формирует готовые CSV/XLSX/JSON выгрузки внешних источников"
|
||||
use_transaction = False
|
||||
|
||||
def add_arguments(self, parser) -> None:
|
||||
super().add_arguments(parser)
|
||||
parser.add_argument(
|
||||
"--all-years",
|
||||
action="store_true",
|
||||
help="Все доступные годы в отдельном каталоге all-years",
|
||||
)
|
||||
|
||||
def execute_command(self, *args, **options) -> str:
|
||||
generation = build_source_record_export_artifacts()
|
||||
generation = build_source_record_export_artifacts(
|
||||
all_years=options["all_years"]
|
||||
)
|
||||
rendered = json.dumps(
|
||||
{
|
||||
"generation_id": generation.generation_id,
|
||||
|
||||
@@ -524,7 +524,7 @@ class SourceRecordExportGeneration:
|
||||
|
||||
generation_id: str
|
||||
generated_at: str
|
||||
export_year: int
|
||||
export_year: int | None
|
||||
artifacts: tuple[SourceRecordExportArtifact, ...]
|
||||
records_count: int
|
||||
|
||||
@@ -594,16 +594,19 @@ def build_source_record_export_artifacts(
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
export_directory: str | Path | None = None,
|
||||
all_years: bool = False,
|
||||
) -> SourceRecordExportGeneration:
|
||||
"""Build all files on disk and atomically publish a new generation."""
|
||||
"""Build annual or all-years files and atomically publish their generation."""
|
||||
|
||||
root_directory = _resolve_export_directory(export_directory)
|
||||
if all_years:
|
||||
root_directory /= "all-years"
|
||||
generations_directory = root_directory / GENERATION_DIRECTORY_NAME
|
||||
root_directory.mkdir(parents=True, exist_ok=True)
|
||||
generations_directory.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
|
||||
export_year = _export_year(generated_at_datetime)
|
||||
export_year = None if all_years else _export_year(generated_at_datetime)
|
||||
generation_id = (
|
||||
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
|
||||
)
|
||||
@@ -943,13 +946,13 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str
|
||||
def _source_model_queryset(
|
||||
model_spec: SourceModelExportSpec,
|
||||
*,
|
||||
export_year: int,
|
||||
export_year: int | None,
|
||||
) -> QuerySet:
|
||||
queryset = cast(
|
||||
QuerySet,
|
||||
model_spec.model.objects.select_related("organization").order_by(),
|
||||
)
|
||||
if model_spec.all_history:
|
||||
if export_year is None or model_spec.all_history:
|
||||
return queryset
|
||||
if model_spec.export_year_lookup:
|
||||
return queryset.filter(
|
||||
@@ -971,7 +974,7 @@ def _source_model_queryset(
|
||||
def _iter_source_model_records(
|
||||
model_spec: SourceModelExportSpec,
|
||||
*,
|
||||
export_year: int,
|
||||
export_year: int | None,
|
||||
) -> Iterator[Any]:
|
||||
iterator = _source_model_queryset(
|
||||
model_spec,
|
||||
@@ -986,7 +989,7 @@ def _iter_source_model_records(
|
||||
Prefetch(
|
||||
related_name,
|
||||
queryset=FinancialReportLine.objects.filter(
|
||||
year=export_year
|
||||
**({"year": export_year} if export_year is not None else {})
|
||||
).order_by(),
|
||||
)
|
||||
if related_name == "lines"
|
||||
@@ -1019,7 +1022,7 @@ def _spool_source_group_rows(
|
||||
*,
|
||||
source_spec: SourceGroupExportSpec,
|
||||
output_path: Path,
|
||||
export_year: int,
|
||||
export_year: int | None,
|
||||
) -> tuple[list[str], int]:
|
||||
records_count = 0
|
||||
with output_path.open("w", encoding="utf-8", newline="") as output:
|
||||
@@ -1405,6 +1408,9 @@ def _generation_manifest_payload(
|
||||
"generation_id": generation.generation_id,
|
||||
"generated_at": generation.generated_at,
|
||||
"export_year": generation.export_year,
|
||||
"export_scope": "all_years"
|
||||
if generation.export_year is None
|
||||
else "current_year",
|
||||
"records_count": generation.records_count,
|
||||
"artifacts_count": generation.artifacts_count,
|
||||
"files_count": generation.files_count,
|
||||
@@ -1425,6 +1431,19 @@ def _generation_manifest_payload(
|
||||
}
|
||||
|
||||
|
||||
def _manifest_export_year(payload: dict[str, Any]) -> int | None:
|
||||
raw_year = payload["export_year"]
|
||||
scope = payload.get("export_scope", "current_year")
|
||||
if scope == "all_years" and raw_year is None:
|
||||
return None
|
||||
if scope != "current_year" or raw_year is None or isinstance(raw_year, bool):
|
||||
raise ValueError("Source-record export scope is invalid.")
|
||||
year = int(raw_year)
|
||||
if not 1 <= year <= 9999:
|
||||
raise ValueError("Source-record export year is invalid.")
|
||||
return year
|
||||
|
||||
|
||||
def _generation_from_manifest(
|
||||
payload: Any,
|
||||
*,
|
||||
@@ -1436,9 +1455,7 @@ def _generation_from_manifest(
|
||||
generation_id = str(payload["generation_id"])
|
||||
generated_at = str(payload["generated_at"])
|
||||
datetime.fromisoformat(generated_at)
|
||||
export_year = int(payload["export_year"])
|
||||
if not 1 <= export_year <= 9999:
|
||||
raise ValueError("Source-record export year is invalid.")
|
||||
export_year = _manifest_export_year(payload)
|
||||
records_count = int(payload["records_count"])
|
||||
artifact_payloads = payload["artifacts"]
|
||||
if not isinstance(artifact_payloads, list):
|
||||
|
||||
@@ -17,6 +17,16 @@ logger = logging.getLogger(__name__)
|
||||
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
||||
"""Build and atomically publish the nightly external-data export matrix."""
|
||||
|
||||
return _refresh_source_record_export_artifacts(self, all_years=False)
|
||||
|
||||
|
||||
@shared_task(bind=True, base=CorePeriodicTask)
|
||||
def refresh_all_years_source_record_export_artifacts(self) -> dict:
|
||||
"""Publish all available years independently of the annual generation."""
|
||||
return _refresh_source_record_export_artifacts(self, all_years=True)
|
||||
|
||||
|
||||
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
|
||||
lock_key = getattr(
|
||||
settings,
|
||||
"SOURCE_RECORD_EXPORT_LOCK_KEY",
|
||||
@@ -26,11 +36,13 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
||||
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
|
||||
)
|
||||
if not cache.add(lock_key, "1", timeout=lock_ttl):
|
||||
if all_years:
|
||||
raise task.retry(countdown=60, max_retries=lock_ttl // 60 + 2)
|
||||
logger.info("Source-record export generation skipped: lock is already held")
|
||||
return {"status": "skipped", "reason": "locked"}
|
||||
|
||||
try:
|
||||
generation = build_source_record_export_artifacts()
|
||||
generation = build_source_record_export_artifacts(all_years=all_years)
|
||||
result = {
|
||||
"status": "success",
|
||||
"generation_id": generation.generation_id,
|
||||
|
||||
Reference in New Issue
Block a user