fix: preserve source export record counts
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 30s
CI/CD Pipeline / Build and Push Images (push) Successful in 34s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 32s

This commit is contained in:
2026-08-04 12:24:09 +02:00
parent de5c1aa4cc
commit eaf8a18f8b
3 changed files with 39 additions and 32 deletions

View File

@@ -43,7 +43,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000
DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60
SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket"
SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}")
EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile(
EXPORT_PROVIDER_URL_PATTERN = re.compile(
r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*",
flags=re.IGNORECASE,
)
EXPORT_PROVIDER_NAME_PATTERN = re.compile(
r"(?<![A-Za-zА-Яа-яЁё0-9])(?:checko(?:\.ru)?|чекало|чекко|чеко)"
r"(?![A-Za-zА-Яа-яЁё0-9])",
flags=re.IGNORECASE,
@@ -535,8 +539,6 @@ def _spool_source_group_rows(
source_group=source_group,
include_financial_lines=include_financial_lines,
):
if _is_excluded_export_provider_record(record):
continue
row = _build_record_row(
record,
include_financial_lines=include_financial_lines,
@@ -719,7 +721,10 @@ def _build_record_row(
"updated_at": record.updated_at,
**_flatten_payload(record.payload),
}
serialized_row = {key: _serialize_json_value(value) for key, value in row.items()}
public_row = _sanitize_export_provider_value(row)
serialized_row = {
key: _serialize_json_value(value) for key, value in public_row.items()
}
if include_financial_lines:
serialized_row["financial_lines"] = [
@@ -732,30 +737,19 @@ def _build_record_row(
return serialized_row
def _is_excluded_export_provider_record(record: OrganizationSourceRecord) -> bool:
"""Return whether a provider-backed record must stay out of public files."""
values = (
record.source,
record.external_id,
record.title,
record.url,
record.payload,
)
return any(_contains_excluded_export_provider(value) for value in values)
def _contains_excluded_export_provider(value: object) -> bool:
def _sanitize_export_provider_value(value: Any) -> Any:
"""Remove provider wording from files without changing stored source data."""
if isinstance(value, str):
return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None
sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value)
return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized)
if isinstance(value, dict):
return any(
_contains_excluded_export_provider(key)
or _contains_excluded_export_provider(item)
return {
_sanitize_export_provider_value(key): _sanitize_export_provider_value(item)
for key, item in value.items()
)
}
if isinstance(value, list | tuple):
return any(_contains_excluded_export_provider(item) for item in value)
return False
return [_sanitize_export_provider_value(item) for item in value]
return value
def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]: