fix: scope export freshness and retries to each source group
Some checks failed
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Tests and lint (push) Has been cancelled
Some checks failed
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Tests and lint (push) Has been cancelled
This commit is contained in:
@@ -2,23 +2,44 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
from uuid import uuid4
|
||||
|
||||
from django.core.cache import cache
|
||||
|
||||
SOURCE_DATA_CACHE_VERSION_KEY = "organizations:source_data:version"
|
||||
SOURCE_DATA_ALL_GROUPS_VERSION_KEY = "organizations:source_data:all_groups_version"
|
||||
|
||||
|
||||
def get_source_data_cache_version() -> str:
|
||||
"""Return current cache version for aggregates derived from source records."""
|
||||
def get_source_data_cache_version(source_group: str | None = None) -> str:
|
||||
"""Return the global aggregate version or one group's export revision."""
|
||||
if source_group is not None:
|
||||
# Bulk maintenance without a known group invalidates every group. Normal
|
||||
# ingestion changes only its own group, while legacy aggregate readers
|
||||
# continue to use the global version below.
|
||||
keys = [SOURCE_DATA_ALL_GROUPS_VERSION_KEY, _group_version_key(source_group)]
|
||||
versions = cache.get_many(keys)
|
||||
value = ":".join(
|
||||
[source_group, *(str(versions.get(k) or "initial") for k in keys)]
|
||||
)
|
||||
return sha256(value.encode()).hexdigest()
|
||||
return str(cache.get(SOURCE_DATA_CACHE_VERSION_KEY) or "initial")
|
||||
|
||||
|
||||
def invalidate_source_data_cache() -> None:
|
||||
"""Invalidate aggregates derived from organization source records."""
|
||||
def _group_version_key(source_group: str) -> str:
|
||||
return f"{SOURCE_DATA_CACHE_VERSION_KEY}:group:{source_group}"
|
||||
|
||||
|
||||
def invalidate_source_data_cache(source_group: str | None = None) -> None:
|
||||
"""Invalidate global aggregates and one source group (all when omitted)."""
|
||||
version = uuid4()
|
||||
cache.set(
|
||||
SOURCE_DATA_CACHE_VERSION_KEY,
|
||||
version.hex if hasattr(version, "hex") else str(version),
|
||||
value = version.hex if hasattr(version, "hex") else str(version)
|
||||
scope_key = (
|
||||
_group_version_key(source_group)
|
||||
if source_group is not None
|
||||
else SOURCE_DATA_ALL_GROUPS_VERSION_KEY
|
||||
)
|
||||
cache.set_many(
|
||||
{SOURCE_DATA_CACHE_VERSION_KEY: value, scope_key: value},
|
||||
timeout=None,
|
||||
)
|
||||
|
||||
@@ -196,7 +196,11 @@ def summary(entry, state=None, generation=None, revision=None):
|
||||
build["state"] == "failed"
|
||||
or generation.created_at < timezone.now() - timedelta(hours=36)
|
||||
or generation.data_revision
|
||||
!= (revision if revision is not None else get_source_data_cache_version())
|
||||
!= (
|
||||
revision
|
||||
if revision is not None
|
||||
else get_source_data_cache_version(entry["source_group"])
|
||||
)
|
||||
)
|
||||
)
|
||||
return {
|
||||
@@ -242,7 +246,6 @@ def catalog():
|
||||
)
|
||||
}
|
||||
entries = registry()
|
||||
revision = get_source_data_cache_version()
|
||||
sources = [
|
||||
summary(
|
||||
entry,
|
||||
@@ -250,7 +253,7 @@ def catalog():
|
||||
generations.get(str(states[group].current_generation))
|
||||
if group in states
|
||||
else None,
|
||||
revision,
|
||||
get_source_data_cache_version(group),
|
||||
)
|
||||
for group, entry in entries.items()
|
||||
]
|
||||
@@ -472,7 +475,7 @@ def _spool(group, path):
|
||||
"CREATE TABLE rows (partition TEXT, source TEXT, kind TEXT, uid TEXT, data TEXT)"
|
||||
)
|
||||
with _source_export_read_snapshot():
|
||||
revision = get_source_data_cache_version()
|
||||
revision = get_source_data_cache_version(group)
|
||||
as_of = _timestamp()
|
||||
queryset = (
|
||||
OrganizationSourceRecord.objects.filter(extension__source_group=group)
|
||||
|
||||
@@ -193,7 +193,7 @@ class OrganizationSourceIngestionService:
|
||||
|
||||
cls._refresh_extension_counters(touched_extension_ids)
|
||||
|
||||
invalidate_source_data_cache()
|
||||
invalidate_source_data_cache(source_group=descriptor.source_group)
|
||||
|
||||
return OrganizationSourceIngestionResult(
|
||||
scanned=scanned,
|
||||
|
||||
@@ -178,7 +178,7 @@ def refresh_source_export_catalog(self, source_group=None):
|
||||
cleanup(source_group)
|
||||
from organizations.source_cache import get_source_data_cache_version
|
||||
|
||||
if generation.data_revision != get_source_data_cache_version():
|
||||
if generation.data_revision != get_source_data_cache_version(source_group):
|
||||
raise self.retry(countdown=30, max_retries=1000)
|
||||
return {
|
||||
"status": "success",
|
||||
|
||||
Reference in New Issue
Block a user