fix: preserve source export record counts
This commit is contained in:
@@ -43,7 +43,11 @@ DEFAULT_XLSX_DATA_ROWS_PER_FILE = 100_000
|
||||
DEFAULT_DOWNLOAD_TICKET_TTL_SECONDS = 5 * 60
|
||||
SOURCE_RECORD_EXPORT_TICKET_CACHE_PREFIX = "organizations:source-record-exports:ticket"
|
||||
SOURCE_RECORD_EXPORT_TICKET_PATTERN = re.compile(r"[A-Za-z0-9_-]{43}")
|
||||
EXCLUDED_EXPORT_PROVIDER_PATTERN = re.compile(
|
||||
EXPORT_PROVIDER_URL_PATTERN = re.compile(
|
||||
r"https?://(?:api\.)?checko\.ru[^\s\"'<>]*",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
EXPORT_PROVIDER_NAME_PATTERN = re.compile(
|
||||
r"(?<![A-Za-zА-Яа-яЁё0-9])(?:checko(?:\.ru)?|чекало|чекко|чеко)"
|
||||
r"(?![A-Za-zА-Яа-яЁё0-9])",
|
||||
flags=re.IGNORECASE,
|
||||
@@ -535,8 +539,6 @@ def _spool_source_group_rows(
|
||||
source_group=source_group,
|
||||
include_financial_lines=include_financial_lines,
|
||||
):
|
||||
if _is_excluded_export_provider_record(record):
|
||||
continue
|
||||
row = _build_record_row(
|
||||
record,
|
||||
include_financial_lines=include_financial_lines,
|
||||
@@ -719,7 +721,10 @@ def _build_record_row(
|
||||
"updated_at": record.updated_at,
|
||||
**_flatten_payload(record.payload),
|
||||
}
|
||||
serialized_row = {key: _serialize_json_value(value) for key, value in row.items()}
|
||||
public_row = _sanitize_export_provider_value(row)
|
||||
serialized_row = {
|
||||
key: _serialize_json_value(value) for key, value in public_row.items()
|
||||
}
|
||||
|
||||
if include_financial_lines:
|
||||
serialized_row["financial_lines"] = [
|
||||
@@ -732,30 +737,19 @@ def _build_record_row(
|
||||
return serialized_row
|
||||
|
||||
|
||||
def _is_excluded_export_provider_record(record: OrganizationSourceRecord) -> bool:
|
||||
"""Return whether a provider-backed record must stay out of public files."""
|
||||
values = (
|
||||
record.source,
|
||||
record.external_id,
|
||||
record.title,
|
||||
record.url,
|
||||
record.payload,
|
||||
)
|
||||
return any(_contains_excluded_export_provider(value) for value in values)
|
||||
|
||||
|
||||
def _contains_excluded_export_provider(value: object) -> bool:
|
||||
def _sanitize_export_provider_value(value: Any) -> Any:
|
||||
"""Remove provider wording from files without changing stored source data."""
|
||||
if isinstance(value, str):
|
||||
return EXCLUDED_EXPORT_PROVIDER_PATTERN.search(value) is not None
|
||||
sanitized = EXPORT_PROVIDER_URL_PATTERN.sub("external-source", value)
|
||||
return EXPORT_PROVIDER_NAME_PATTERN.sub("external-source", sanitized)
|
||||
if isinstance(value, dict):
|
||||
return any(
|
||||
_contains_excluded_export_provider(key)
|
||||
or _contains_excluded_export_provider(item)
|
||||
return {
|
||||
_sanitize_export_provider_value(key): _sanitize_export_provider_value(item)
|
||||
for key, item in value.items()
|
||||
)
|
||||
}
|
||||
if isinstance(value, list | tuple):
|
||||
return any(_contains_excluded_export_provider(item) for item in value)
|
||||
return False
|
||||
return [_sanitize_export_provider_value(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _flatten_payload(value: Any, *, prefix: str = "payload") -> dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user