fix: scope export freshness and retries to each source group
Some checks failed
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Tests and lint (push) Has been cancelled
Some checks failed
Mostovik Backend CI/CD / Build linux/amd64 release images (push) Has been cancelled
Mostovik Backend CI/CD / Deploy dev (push) Has been cancelled
Mostovik Backend CI/CD / Deploy and verify internal main (push) Has been cancelled
Mostovik Backend CI/CD / Tests and lint (push) Has been cancelled
This commit is contained in:
@@ -46,6 +46,82 @@ GROUP = "government_procurements"
|
||||
|
||||
|
||||
class SourceExportCatalogTest(APITestCase):
|
||||
def test_unrelated_ingestion_during_export_does_not_retry_or_mark_stale(self):
|
||||
from organizations import source_export_catalog
|
||||
from organizations.source_cache import (
|
||||
get_source_data_cache_version,
|
||||
invalidate_source_data_cache,
|
||||
)
|
||||
from organizations.tasks import refresh_source_export_catalog
|
||||
|
||||
self.records(3)
|
||||
render = source_export_catalog._render_parts
|
||||
global_before = get_source_data_cache_version()
|
||||
|
||||
def render_while_vacancies_change(*args, **kwargs):
|
||||
invalidate_source_data_cache(source_group="vacancies")
|
||||
return render(*args, **kwargs)
|
||||
|
||||
with patch.object(
|
||||
source_export_catalog,
|
||||
"_render_parts",
|
||||
side_effect=render_while_vacancies_change,
|
||||
), patch.object(refresh_source_export_catalog, "retry") as retry:
|
||||
result = refresh_source_export_catalog.run(source_group=GROUP)
|
||||
self.assertEqual(result["status"], "success")
|
||||
retry.assert_not_called()
|
||||
self.assertNotEqual(global_before, get_source_data_cache_version())
|
||||
source = next(s for s in catalog()["sources"] if s["source_group"] == GROUP)
|
||||
self.assertFalse(source["is_stale"])
|
||||
|
||||
def test_same_source_change_during_export_retries_after_publishing_snapshot(self):
|
||||
from celery.exceptions import Retry
|
||||
from organizations import source_export_catalog
|
||||
from organizations.source_cache import invalidate_source_data_cache
|
||||
from organizations.tasks import refresh_source_export_catalog
|
||||
|
||||
self.records(3)
|
||||
render = source_export_catalog._render_parts
|
||||
|
||||
def render_while_source_changes(*args, **kwargs):
|
||||
invalidate_source_data_cache(source_group=GROUP)
|
||||
return render(*args, **kwargs)
|
||||
|
||||
with patch.object(
|
||||
source_export_catalog,
|
||||
"_render_parts",
|
||||
side_effect=render_while_source_changes,
|
||||
), patch.object(
|
||||
refresh_source_export_catalog, "retry", side_effect=Retry()
|
||||
) as retry:
|
||||
with self.assertRaises(Retry):
|
||||
refresh_source_export_catalog.run(source_group=GROUP)
|
||||
retry.assert_called_once_with(countdown=30, max_retries=1000)
|
||||
self.assertEqual(SourceExportGeneration.objects.count(), 1)
|
||||
source = next(s for s in catalog()["sources"] if s["source_group"] == GROUP)
|
||||
self.assertEqual(source["state"], "ready")
|
||||
self.assertTrue(source["is_stale"])
|
||||
|
||||
def test_bulk_invalidation_stales_all_groups_after_scoped_ingestion(self):
|
||||
from organizations.source_cache import (
|
||||
get_source_data_cache_version,
|
||||
invalidate_source_data_cache,
|
||||
)
|
||||
|
||||
invalidate_source_data_cache(source_group=GROUP)
|
||||
group_before = get_source_data_cache_version(GROUP)
|
||||
vacancies_before = get_source_data_cache_version("vacancies")
|
||||
self.records(3)
|
||||
build_source(GROUP)
|
||||
invalidate_source_data_cache()
|
||||
self.assertNotEqual(group_before, get_source_data_cache_version(GROUP))
|
||||
self.assertNotEqual(
|
||||
vacancies_before, get_source_data_cache_version("vacancies")
|
||||
)
|
||||
self.assertEqual(len(get_source_data_cache_version(GROUP)), 64)
|
||||
source = next(s for s in catalog()["sources"] if s["source_group"] == GROUP)
|
||||
self.assertTrue(source["is_stale"])
|
||||
|
||||
def setUp(self):
|
||||
self.directory = TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
|
||||
@@ -29,6 +29,13 @@ class OrganizationSourceIngestionServiceTest(TestCase):
|
||||
return Organization.objects.create(**kwargs)
|
||||
|
||||
def test_save_generic_records_writes_source_records_without_legacy_rows(self):
|
||||
from organizations.source_cache import get_source_data_cache_version
|
||||
|
||||
global_before = get_source_data_cache_version()
|
||||
source_before = get_source_data_cache_version(SourceGroup.DEFENSE_SUPPLIERS)
|
||||
other_before = get_source_data_cache_version(
|
||||
SourceGroup.GOVERNMENT_PROCUREMENTS
|
||||
)
|
||||
self._directory_organization(
|
||||
name='ООО "ГОЗ"',
|
||||
inn="7707083803",
|
||||
@@ -54,6 +61,14 @@ class OrganizationSourceIngestionServiceTest(TestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(result.scanned, 1)
|
||||
self.assertNotEqual(global_before, get_source_data_cache_version())
|
||||
self.assertNotEqual(
|
||||
source_before, get_source_data_cache_version(SourceGroup.DEFENSE_SUPPLIERS)
|
||||
)
|
||||
self.assertEqual(
|
||||
other_before,
|
||||
get_source_data_cache_version(SourceGroup.GOVERNMENT_PROCUREMENTS),
|
||||
)
|
||||
self.assertEqual(result.created_records, 1)
|
||||
self.assertEqual(result.updated_records, 0)
|
||||
self.assertEqual(result.created_extensions, 1)
|
||||
|
||||
Reference in New Issue
Block a user