Compare commits
4 Commits
hotfix/jso
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e712494c3f | ||
| bad64fe001 | |||
|
|
a00ecec936 | ||
| 448b2c2f7c |
15
docs/all-years-source-exports.md
Normal file
15
docs/all-years-source-exports.md
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Выгрузка за все годы
|
||||||
|
|
||||||
|
Годовая задача `organizations.tasks.refresh_source_record_export_artifacts` сохраняет прежнее поведение.
|
||||||
|
|
||||||
|
Новая задача `organizations.tasks.refresh_all_years_source_record_export_artifacts` собирает все доступные записи каждого поддерживаемого источника без ограничения по году, включая все финансовые строки. Для источников с заменяемыми snapshot выгружается текущее опубликованное состояние: удалённые прежние snapshot не восстанавливаются.
|
||||||
|
|
||||||
|
Результат размещается в `SOURCE_RECORD_EXPORT_DIRECTORY/all-years`. У него отдельные `current.json`, `generations` и очистка старых поколений. Публикация атомарна; незавершённая сборка не заменяет опубликованные файлы. Общий lock с годовой задачей ограничивает одновременную нагрузку. Manifest содержит `export_scope: all_years` и `export_year: null`; годовой manifest — `export_scope: current_year` и числовой год. Старые годовые manifest без `export_scope` продолжают читаться.
|
||||||
|
|
||||||
|
Ручной запуск:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python src/manage.py build_source_record_exports --all-years
|
||||||
|
```
|
||||||
|
|
||||||
|
Периодический запуск настраивается отдельной записью django-celery-beat либо системным таймером на контуре без Beat. Задачу следует ставить после годовой сборки. Имена файлов, группировка JSON по ОКПО и форматы CSV/XLSX/JSON сохраняются. API скачивания продолжает отдавать годовой пакет; доступ к полному пакету предоставляется отдельно.
|
||||||
15
docs/dashboard-performance.md
Normal file
15
docs/dashboard-performance.md
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Повторные расчеты dashboard
|
||||||
|
|
||||||
|
`/api/v1/parsers/dashboard/` использует один расчет соответствий организаций
|
||||||
|
источникам для покрытия и аналитики обогащения. Последние логи также читаются
|
||||||
|
один раз на запрос. Между запросами результаты не кешируются.
|
||||||
|
|
||||||
|
На одном снимке данных внутреннего main число SQL-запросов уменьшилось с 51
|
||||||
|
до 37, время обработчика — с 1,76 до 1,14 секунды. Полные JSON-ответы до и после
|
||||||
|
совпали. Сравнение выполнено в отдельном процессе, в транзакции REPEATABLE READ,
|
||||||
|
READ ONLY, с фиксированным временем. Сеть и аутентификация не измерялись.
|
||||||
|
|
||||||
|
В этой версии main нет дорогого поиска последних фоновых задач из dev.
|
||||||
|
Выборка карточек источников этим патчем не меняется. Миграций, новых зависимостей
|
||||||
|
и настроек нет. Регрессия в `test_views.py` проверяет отсутствие повторных
|
||||||
|
обходов записей источников вместе с прежними значениями аналитики.
|
||||||
@@ -1986,8 +1986,9 @@ def _registry_source_matrix_rows(
|
|||||||
return matrix_rows
|
return matrix_rows
|
||||||
|
|
||||||
|
|
||||||
def _registry_data_coverage() -> dict:
|
def _registry_data_coverage(
|
||||||
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
registry_organization_ids: set[str], source_matches: dict[str, set[str]]
|
||||||
|
) -> dict:
|
||||||
total_organizations = len(registry_organization_ids)
|
total_organizations = len(registry_organization_ids)
|
||||||
descriptor_titles: dict[str, str] = {}
|
descriptor_titles: dict[str, str] = {}
|
||||||
for descriptor in PARSER_SOURCES.values():
|
for descriptor in PARSER_SOURCES.values():
|
||||||
@@ -1998,9 +1999,7 @@ def _registry_data_coverage() -> dict:
|
|||||||
for source, _label in ParserLoadLog.Source.choices:
|
for source, _label in ParserLoadLog.Source.choices:
|
||||||
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
|
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
|
||||||
continue
|
continue
|
||||||
organizations_count = len(
|
organizations_count = len(source_matches.get(source, set()))
|
||||||
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
|
|
||||||
)
|
|
||||||
coverage_percent = (
|
coverage_percent = (
|
||||||
round(organizations_count / total_organizations * 100, 1)
|
round(organizations_count / total_organizations * 100, 1)
|
||||||
if total_organizations
|
if total_organizations
|
||||||
@@ -2021,10 +2020,14 @@ def _registry_data_coverage() -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
|
def _registry_enrichment_analytics(
|
||||||
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
jobs,
|
||||||
|
schedules: list[dict],
|
||||||
|
registry_organization_ids: set[str],
|
||||||
|
source_matches: dict[str, set[str]],
|
||||||
|
latest_loads: list[ParserLoadLog],
|
||||||
|
) -> dict:
|
||||||
total_organizations = len(registry_organization_ids)
|
total_organizations = len(registry_organization_ids)
|
||||||
source_matches = _source_registry_matches(by_inn, by_ogrn)
|
|
||||||
descriptor_titles: dict[str, str] = {}
|
descriptor_titles: dict[str, str] = {}
|
||||||
for descriptor in PARSER_SOURCES.values():
|
for descriptor in PARSER_SOURCES.values():
|
||||||
descriptor_titles.setdefault(descriptor.source, descriptor.title)
|
descriptor_titles.setdefault(descriptor.source, descriptor.title)
|
||||||
@@ -2113,7 +2116,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
|
|||||||
enrichment_sources=enrichment_sources,
|
enrichment_sources=enrichment_sources,
|
||||||
)
|
)
|
||||||
|
|
||||||
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
|
|
||||||
recent_success = sum(
|
recent_success = sum(
|
||||||
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
|
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
|
||||||
)
|
)
|
||||||
@@ -2948,6 +2950,9 @@ class ParserDashboardDataView(APIView):
|
|||||||
_periodic_task_to_dict(task)
|
_periodic_task_to_dict(task)
|
||||||
for task in _parser_periodic_tasks_for_user(request.user)
|
for task in _parser_periodic_tasks_for_user(request.user)
|
||||||
]
|
]
|
||||||
|
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
|
||||||
|
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
|
||||||
|
source_matches = _source_registry_matches(by_inn, by_ogrn)
|
||||||
return api_response(
|
return api_response(
|
||||||
{
|
{
|
||||||
"sources": sources,
|
"sources": sources,
|
||||||
@@ -2960,13 +2965,18 @@ class ParserDashboardDataView(APIView):
|
|||||||
"schedules": ParserScheduleSerializer(schedules, many=True).data,
|
"schedules": ParserScheduleSerializer(schedules, many=True).data,
|
||||||
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
|
"jobs": BackgroundJobListSerializer(jobs, many=True).data,
|
||||||
"source_counts": source_counts,
|
"source_counts": source_counts,
|
||||||
"registry_data_coverage": _registry_data_coverage(),
|
"registry_data_coverage": _registry_data_coverage(
|
||||||
|
registry_ids, source_matches
|
||||||
|
),
|
||||||
"registry_enrichment_analytics": _registry_enrichment_analytics(
|
"registry_enrichment_analytics": _registry_enrichment_analytics(
|
||||||
jobs,
|
jobs,
|
||||||
schedules,
|
schedules,
|
||||||
|
registry_ids,
|
||||||
|
source_matches,
|
||||||
|
load_logs,
|
||||||
),
|
),
|
||||||
"load_logs": ParserLoadLogSerializer(
|
"load_logs": ParserLoadLogSerializer(
|
||||||
ParserLoadLog.objects.all().order_by("-created_at")[:30],
|
load_logs,
|
||||||
many=True,
|
many=True,
|
||||||
).data,
|
).data,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,8 +15,18 @@ class Command(BaseAppCommand):
|
|||||||
help = "Формирует готовые CSV/XLSX/JSON выгрузки источников"
|
help = "Формирует готовые CSV/XLSX/JSON выгрузки источников"
|
||||||
use_transaction = False
|
use_transaction = False
|
||||||
|
|
||||||
|
def add_arguments(self, parser) -> None:
|
||||||
|
super().add_arguments(parser)
|
||||||
|
parser.add_argument(
|
||||||
|
"--all-years",
|
||||||
|
action="store_true",
|
||||||
|
help="Все доступные годы в отдельном каталоге all-years",
|
||||||
|
)
|
||||||
|
|
||||||
def execute_command(self, *args, **options) -> str:
|
def execute_command(self, *args, **options) -> str:
|
||||||
generation = build_source_record_export_artifacts()
|
generation = build_source_record_export_artifacts(
|
||||||
|
all_years=options["all_years"]
|
||||||
|
)
|
||||||
rendered = json.dumps(
|
rendered = json.dumps(
|
||||||
{
|
{
|
||||||
"generation_id": generation.generation_id,
|
"generation_id": generation.generation_id,
|
||||||
|
|||||||
@@ -127,7 +127,7 @@ class SourceRecordExportGeneration:
|
|||||||
|
|
||||||
generation_id: str
|
generation_id: str
|
||||||
generated_at: str
|
generated_at: str
|
||||||
export_year: int
|
export_year: int | None
|
||||||
artifacts: tuple[SourceRecordExportArtifact, ...]
|
artifacts: tuple[SourceRecordExportArtifact, ...]
|
||||||
records_count: int
|
records_count: int
|
||||||
|
|
||||||
@@ -197,16 +197,19 @@ def build_source_record_export_artifacts(
|
|||||||
*,
|
*,
|
||||||
now: datetime | None = None,
|
now: datetime | None = None,
|
||||||
export_directory: str | Path | None = None,
|
export_directory: str | Path | None = None,
|
||||||
|
all_years: bool = False,
|
||||||
) -> SourceRecordExportGeneration:
|
) -> SourceRecordExportGeneration:
|
||||||
"""Build all files on disk and atomically publish a new generation."""
|
"""Build annual or all-years files and atomically publish their generation."""
|
||||||
|
|
||||||
root_directory = _resolve_export_directory(export_directory)
|
root_directory = _resolve_export_directory(export_directory)
|
||||||
|
if all_years:
|
||||||
|
root_directory /= "all-years"
|
||||||
generations_directory = root_directory / GENERATION_DIRECTORY_NAME
|
generations_directory = root_directory / GENERATION_DIRECTORY_NAME
|
||||||
root_directory.mkdir(parents=True, exist_ok=True)
|
root_directory.mkdir(parents=True, exist_ok=True)
|
||||||
generations_directory.mkdir(parents=True, exist_ok=True)
|
generations_directory.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
|
generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
|
||||||
export_year = _export_year(generated_at_datetime)
|
export_year = None if all_years else _export_year(generated_at_datetime)
|
||||||
generation_id = (
|
generation_id = (
|
||||||
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
|
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
|
||||||
)
|
)
|
||||||
@@ -545,7 +548,7 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str
|
|||||||
def _source_group_queryset(
|
def _source_group_queryset(
|
||||||
source_group: str,
|
source_group: str,
|
||||||
*,
|
*,
|
||||||
export_year: int,
|
export_year: int | None,
|
||||||
) -> QuerySet[OrganizationSourceRecord]:
|
) -> QuerySet[OrganizationSourceRecord]:
|
||||||
queryset = (
|
queryset = (
|
||||||
OrganizationSourceRecord.objects.filter(extension__source_group=source_group)
|
OrganizationSourceRecord.objects.filter(extension__source_group=source_group)
|
||||||
@@ -555,6 +558,8 @@ def _source_group_queryset(
|
|||||||
# source groups with millions of rows.
|
# source groups with millions of rows.
|
||||||
.order_by()
|
.order_by()
|
||||||
)
|
)
|
||||||
|
if export_year is None:
|
||||||
|
return queryset
|
||||||
if source_group == FINANCIAL_SOURCE_GROUP:
|
if source_group == FINANCIAL_SOURCE_GROUP:
|
||||||
return queryset.filter(financial_lines__year=export_year).distinct()
|
return queryset.filter(financial_lines__year=export_year).distinct()
|
||||||
|
|
||||||
@@ -581,7 +586,7 @@ def _iter_source_records(
|
|||||||
*,
|
*,
|
||||||
source_group: str,
|
source_group: str,
|
||||||
include_financial_lines: bool,
|
include_financial_lines: bool,
|
||||||
export_year: int,
|
export_year: int | None,
|
||||||
) -> Iterator[OrganizationSourceRecord]:
|
) -> Iterator[OrganizationSourceRecord]:
|
||||||
iterator = _source_group_queryset(
|
iterator = _source_group_queryset(
|
||||||
source_group,
|
source_group,
|
||||||
@@ -597,7 +602,7 @@ def _iter_source_records(
|
|||||||
Prefetch(
|
Prefetch(
|
||||||
"financial_lines",
|
"financial_lines",
|
||||||
queryset=OrganizationSourceFinancialLine.objects.filter(
|
queryset=OrganizationSourceFinancialLine.objects.filter(
|
||||||
year=export_year
|
**({"year": export_year} if export_year is not None else {})
|
||||||
).order_by(),
|
).order_by(),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -608,7 +613,7 @@ def _spool_source_group_rows(
|
|||||||
*,
|
*,
|
||||||
source_group: str,
|
source_group: str,
|
||||||
output_path: Path,
|
output_path: Path,
|
||||||
export_year: int,
|
export_year: int | None,
|
||||||
) -> tuple[list[str], int]:
|
) -> tuple[list[str], int]:
|
||||||
include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP
|
include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP
|
||||||
payload_headers: set[str] = set()
|
payload_headers: set[str] = set()
|
||||||
@@ -953,6 +958,9 @@ def _generation_manifest_payload(
|
|||||||
"generation_id": generation.generation_id,
|
"generation_id": generation.generation_id,
|
||||||
"generated_at": generation.generated_at,
|
"generated_at": generation.generated_at,
|
||||||
"export_year": generation.export_year,
|
"export_year": generation.export_year,
|
||||||
|
"export_scope": "all_years"
|
||||||
|
if generation.export_year is None
|
||||||
|
else "current_year",
|
||||||
"records_count": generation.records_count,
|
"records_count": generation.records_count,
|
||||||
"artifacts_count": generation.artifacts_count,
|
"artifacts_count": generation.artifacts_count,
|
||||||
"files_count": generation.files_count,
|
"files_count": generation.files_count,
|
||||||
@@ -973,6 +981,19 @@ def _generation_manifest_payload(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _manifest_export_year(payload: dict[str, Any]) -> int | None:
|
||||||
|
raw_year = payload["export_year"]
|
||||||
|
scope = payload.get("export_scope", "current_year")
|
||||||
|
if scope == "all_years" and raw_year is None:
|
||||||
|
return None
|
||||||
|
if scope != "current_year" or raw_year is None or isinstance(raw_year, bool):
|
||||||
|
raise ValueError("Source-record export scope is invalid.")
|
||||||
|
year = int(raw_year)
|
||||||
|
if not 1 <= year <= 9999:
|
||||||
|
raise ValueError("Source-record export year is invalid.")
|
||||||
|
return year
|
||||||
|
|
||||||
|
|
||||||
def _generation_from_manifest(
|
def _generation_from_manifest(
|
||||||
payload: Any,
|
payload: Any,
|
||||||
*,
|
*,
|
||||||
@@ -984,9 +1005,7 @@ def _generation_from_manifest(
|
|||||||
generation_id = str(payload["generation_id"])
|
generation_id = str(payload["generation_id"])
|
||||||
generated_at = str(payload["generated_at"])
|
generated_at = str(payload["generated_at"])
|
||||||
datetime.fromisoformat(generated_at)
|
datetime.fromisoformat(generated_at)
|
||||||
export_year = int(payload["export_year"])
|
export_year = _manifest_export_year(payload)
|
||||||
if not 1 <= export_year <= 9999:
|
|
||||||
raise ValueError("Source-record export year is invalid.")
|
|
||||||
records_count = int(payload["records_count"])
|
records_count = int(payload["records_count"])
|
||||||
artifact_payloads = payload["artifacts"]
|
artifact_payloads = payload["artifacts"]
|
||||||
if not isinstance(artifact_payloads, list):
|
if not isinstance(artifact_payloads, list):
|
||||||
|
|||||||
@@ -20,6 +20,16 @@ logger = logging.getLogger(__name__)
|
|||||||
@shared_task(bind=True, base=CorePeriodicTask)
|
@shared_task(bind=True, base=CorePeriodicTask)
|
||||||
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
||||||
"""Build and atomically publish the nightly source-record export matrix."""
|
"""Build and atomically publish the nightly source-record export matrix."""
|
||||||
|
return _refresh_source_record_export_artifacts(self, all_years=False)
|
||||||
|
|
||||||
|
|
||||||
|
@shared_task(bind=True, base=CorePeriodicTask)
|
||||||
|
def refresh_all_years_source_record_export_artifacts(self) -> dict:
|
||||||
|
"""Publish all available years independently of the annual generation."""
|
||||||
|
return _refresh_source_record_export_artifacts(self, all_years=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
|
||||||
lock_key = getattr(
|
lock_key = getattr(
|
||||||
settings,
|
settings,
|
||||||
"SOURCE_RECORD_EXPORT_LOCK_KEY",
|
"SOURCE_RECORD_EXPORT_LOCK_KEY",
|
||||||
@@ -29,11 +39,13 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
|
|||||||
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
|
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
|
||||||
)
|
)
|
||||||
if not cache.add(lock_key, "1", timeout=lock_ttl):
|
if not cache.add(lock_key, "1", timeout=lock_ttl):
|
||||||
|
if all_years:
|
||||||
|
raise task.retry(countdown=60, max_retries=lock_ttl // 60 + 2)
|
||||||
logger.info("Source-record export generation skipped: lock is already held")
|
logger.info("Source-record export generation skipped: lock is already held")
|
||||||
return {"status": "skipped", "reason": "locked"}
|
return {"status": "skipped", "reason": "locked"}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
generation = build_source_record_export_artifacts()
|
generation = build_source_record_export_artifacts(all_years=all_years)
|
||||||
result = {
|
result = {
|
||||||
"status": "success",
|
"status": "success",
|
||||||
"generation_id": generation.generation_id,
|
"generation_id": generation.generation_id,
|
||||||
|
|||||||
@@ -216,6 +216,126 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
|
|||||||
{export_year},
|
{export_year},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_all_years_generation_is_separate_and_keeps_historical_financial_lines(
|
||||||
|
self,
|
||||||
|
):
|
||||||
|
organization = Organization.objects.create(name="Export", inn="7707083899")
|
||||||
|
extension = PlannedInspectionExtension.objects.create(organization=organization)
|
||||||
|
for year in (2020, 2026):
|
||||||
|
OrganizationSourceRecord.objects.create(
|
||||||
|
extension=extension,
|
||||||
|
record_type="inspection",
|
||||||
|
source="inspections",
|
||||||
|
external_id=str(year),
|
||||||
|
record_date=f"{year}-01-01",
|
||||||
|
)
|
||||||
|
finance = FinancialIndicatorsExtension.objects.create(organization=organization)
|
||||||
|
report = OrganizationSourceRecord.objects.create(
|
||||||
|
extension=finance,
|
||||||
|
record_type="financial_report",
|
||||||
|
source="fns_reports",
|
||||||
|
external_id="full",
|
||||||
|
)
|
||||||
|
for year in (2020, 2026):
|
||||||
|
OrganizationSourceFinancialLine.objects.create(
|
||||||
|
source_record=report,
|
||||||
|
year=year,
|
||||||
|
form_code="1",
|
||||||
|
line_code=str(year),
|
||||||
|
)
|
||||||
|
source = SourceGroup.PLANNED_INSPECTIONS.value
|
||||||
|
annual = build_source_record_export_artifacts(
|
||||||
|
now=datetime(2026, 8, 4, tzinfo=UTC)
|
||||||
|
)
|
||||||
|
annual_manifest = Path(self.export_directory.name) / "current.json"
|
||||||
|
before = annual_manifest.read_bytes()
|
||||||
|
full = build_source_record_export_artifacts(
|
||||||
|
now=datetime(2026, 8, 4, tzinfo=UTC),
|
||||||
|
all_years=True,
|
||||||
|
)
|
||||||
|
self.assertIsNone(full.export_year)
|
||||||
|
self.assertEqual(annual_manifest.read_bytes(), before)
|
||||||
|
full_root = Path(self.export_directory.name) / "all-years"
|
||||||
|
loaded = load_current_source_record_export_generation(
|
||||||
|
export_directory=full_root
|
||||||
|
)
|
||||||
|
self.assertEqual(loaded.generation_id, full.generation_id)
|
||||||
|
self.assertIsNone(loaded.export_year)
|
||||||
|
|
||||||
|
def rows(generation, group):
|
||||||
|
artifact = next(
|
||||||
|
a
|
||||||
|
for a in generation.artifacts
|
||||||
|
if a.source_group == group and a.file_format == "json"
|
||||||
|
)
|
||||||
|
return _source_records(json.loads(artifact.path.read_text()), group)
|
||||||
|
|
||||||
|
self.assertEqual(len(rows(annual, source)), 1)
|
||||||
|
self.assertEqual(len(rows(full, source)), 2)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
line["year"]
|
||||||
|
for line in rows(full, "financial_indicators")[0]["financial_lines"]
|
||||||
|
},
|
||||||
|
{2020, 2026},
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
{
|
||||||
|
line["year"]
|
||||||
|
for line in rows(annual, "financial_indicators")[0]["financial_lines"]
|
||||||
|
},
|
||||||
|
{2026},
|
||||||
|
)
|
||||||
|
# Retention in either scope must not delete the other's files.
|
||||||
|
for _ in range(3):
|
||||||
|
build_source_record_export_artifacts(all_years=True)
|
||||||
|
self.assertTrue(all(a.path.is_file() for a in annual.artifacts))
|
||||||
|
|
||||||
|
def test_all_years_command_does_not_publish_an_annual_manifest(self):
|
||||||
|
call_command("build_source_record_exports", all_years=True, stdout=StringIO())
|
||||||
|
self.assertFalse((Path(self.export_directory.name) / "current.json").exists())
|
||||||
|
full = load_current_source_record_export_generation(
|
||||||
|
export_directory=Path(self.export_directory.name) / "all-years",
|
||||||
|
)
|
||||||
|
self.assertIsNone(full.export_year)
|
||||||
|
|
||||||
|
def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self):
|
||||||
|
from celery.exceptions import Retry
|
||||||
|
from django.core.cache import cache
|
||||||
|
from organizations.tasks import refresh_all_years_source_record_export_artifacts
|
||||||
|
|
||||||
|
lock = "test-full-export-lock"
|
||||||
|
with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock):
|
||||||
|
cache.delete(lock)
|
||||||
|
cache.add(lock, "held", timeout=30)
|
||||||
|
try:
|
||||||
|
with self.assertRaises(Retry):
|
||||||
|
refresh_all_years_source_record_export_artifacts.run()
|
||||||
|
self.assertFalse(
|
||||||
|
(
|
||||||
|
Path(self.export_directory.name) / "all-years/current.json"
|
||||||
|
).exists()
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
cache.delete(lock)
|
||||||
|
result = refresh_all_years_source_record_export_artifacts.run()
|
||||||
|
self.assertEqual(result["status"], "success")
|
||||||
|
self.assertIsNone(result["export_year"])
|
||||||
|
self.assertIsNone(cache.get(lock))
|
||||||
|
self.assertFalse(
|
||||||
|
(Path(self.export_directory.name) / "current.json").exists()
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_manifest_rejects_ambiguous_all_years_scope(self):
|
||||||
|
build_source_record_export_artifacts(all_years=True)
|
||||||
|
directory = Path(self.export_directory.name) / "all-years"
|
||||||
|
manifest_path = directory / "current.json"
|
||||||
|
payload = json.loads(manifest_path.read_text())
|
||||||
|
payload["export_year"] = 2026
|
||||||
|
manifest_path.write_text(json.dumps(payload))
|
||||||
|
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
|
||||||
|
load_current_source_record_export_generation(export_directory=directory)
|
||||||
|
|
||||||
def test_new_calendar_year_requires_a_new_prepared_generation(self):
|
def test_new_calendar_year_requires_a_new_prepared_generation(self):
|
||||||
generated_at = datetime(2026, 12, 31, 23, 59, tzinfo=UTC)
|
generated_at = datetime(2026, 12, 31, 23, 59, tzinfo=UTC)
|
||||||
build_source_record_export_artifacts(now=generated_at)
|
build_source_record_export_artifacts(now=generated_at)
|
||||||
|
|||||||
@@ -20,6 +20,8 @@ from apps.parsers.models import (
|
|||||||
ProcurementRecord,
|
ProcurementRecord,
|
||||||
)
|
)
|
||||||
from django.core.files.uploadedfile import SimpleUploadedFile
|
from django.core.files.uploadedfile import SimpleUploadedFile
|
||||||
|
from django.db import connection
|
||||||
|
from django.test.utils import CaptureQueriesContext
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
from openpyxl import Workbook
|
from openpyxl import Workbook
|
||||||
from organizations.models import Organization, OrganizationSourceRecord
|
from organizations.models import Organization, OrganizationSourceRecord
|
||||||
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
|
|||||||
)
|
)
|
||||||
self.client.force_authenticate(self.user)
|
self.client.force_authenticate(self.user)
|
||||||
|
|
||||||
response = self.client.get("/api/v1/parsers/dashboard/")
|
with CaptureQueriesContext(connection) as queries:
|
||||||
|
response = self.client.get("/api/v1/parsers/dashboard/")
|
||||||
|
|
||||||
|
source_scans = [
|
||||||
|
query["sql"]
|
||||||
|
for query in queries
|
||||||
|
if "SELECT DISTINCT" in query["sql"]
|
||||||
|
and "organizations_source_record" in query["sql"]
|
||||||
|
]
|
||||||
|
self.assertTrue(source_scans)
|
||||||
|
self.assertEqual(len(source_scans), len(set(source_scans)))
|
||||||
|
|
||||||
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
self.assertEqual(response.status_code, status.HTTP_200_OK)
|
||||||
coverage = response.data["data"]["registry_data_coverage"]
|
coverage = response.data["data"]["registry_data_coverage"]
|
||||||
|
|||||||
Reference in New Issue
Block a user