Compare commits

..

4 Commits

Author SHA1 Message Date
Aleksandr Meshchryakov
e712494c3f fix: reuse source coverage calculations on main dashboard
All checks were successful
CI/CD Pipeline / Quality Gate (pull_request) Successful in 1m3s
CI/CD Pipeline / Build and Push Images (pull_request) Successful in 1s
CI/CD Pipeline / Internal Notify (pull_request) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (pull_request) Successful in 1s
CI/CD Pipeline / Quality Gate (push) Successful in 49s
CI/CD Pipeline / Build and Push Images (push) Successful in 1s
CI/CD Pipeline / Internal Notify (push) Successful in 1s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
2026-10-01 13:21:52 +02:00
avm
bad64fe001 Export all available years in a separate scheduled task [skip ci] #no_deploy #no_image
Reviewed hotfix. Manual immutable release with data preservation and runtime verification.
2026-09-28 13:20:07 +03:00
Aleksandr Meshchryakov
a00ecec936 feat: add separate all-years source export task
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 49s
CI/CD Pipeline / Build and Push Images (push) Successful in 1s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
2026-09-28 12:06:09 +02:00
avm
448b2c2f7c Merge pull request 'fix: group JSON source exports by organization (main)' (#42) from hotfix/json-source-export-20260927 into main
All checks were successful
CI/CD Pipeline / Quality Gate (push) Successful in 57s
CI/CD Pipeline / Build and Push Images (push) Successful in 0s
CI/CD Pipeline / Internal Notify (push) Successful in 0s
CI/CD Pipeline / Deploy Dev via Compose (push) Successful in 1s
2026-09-27 11:43:35 +03:00
8 changed files with 237 additions and 24 deletions

View File

@@ -0,0 +1,15 @@
# Выгрузка за все годы
Годовая задача `organizations.tasks.refresh_source_record_export_artifacts` сохраняет прежнее поведение.
Новая задача `organizations.tasks.refresh_all_years_source_record_export_artifacts` собирает все доступные записи каждого поддерживаемого источника без ограничения по году, включая все финансовые строки. Для источников с заменяемыми snapshot выгружается текущее опубликованное состояние: удалённые прежние snapshot не восстанавливаются.
Результат размещается в `SOURCE_RECORD_EXPORT_DIRECTORY/all-years`. У него отдельные `current.json`, `generations` и очистка старых поколений. Публикация атомарна; незавершённая сборка не заменяет опубликованные файлы. Общий lock с годовой задачей ограничивает одновременную нагрузку. Manifest содержит `export_scope: all_years` и `export_year: null`; годовой manifest — `export_scope: current_year` и числовой год. Старые годовые manifest без `export_scope` продолжают читаться.
Ручной запуск:
```sh
python src/manage.py build_source_record_exports --all-years
```
Периодический запуск настраивается отдельной записью django-celery-beat либо системным таймером на контуре без Beat. Задачу следует ставить после годовой сборки. Имена файлов, группировка JSON по ОКПО и форматы CSV/XLSX/JSON сохраняются. API скачивания продолжает отдавать годовой пакет; доступ к полному пакету предоставляется отдельно.

View File

@@ -0,0 +1,15 @@
# Повторные расчеты dashboard
`/api/v1/parsers/dashboard/` использует один расчет соответствий организаций
источникам для покрытия и аналитики обогащения. Последние логи также читаются
один раз на запрос. Между запросами результаты не кешируются.
На одном снимке данных внутреннего main число SQL-запросов уменьшилось с 51
до 37, время обработчика — с 1,76 до 1,14 секунды. Полные JSON-ответы до и после
совпали. Сравнение выполнено в отдельном процессе, в транзакции REPEATABLE READ,
READ ONLY, с фиксированным временем. Сеть и аутентификация не измерялись.
В этой версии main нет дорогого поиска последних фоновых задач из dev.
Выборка карточек источников этим патчем не меняется. Миграций, новых зависимостей
и настроек нет. Регрессия в `test_views.py` проверяет отсутствие повторных
обходов записей источников вместе с прежними значениями аналитики.

View File

@@ -1986,8 +1986,9 @@ def _registry_source_matrix_rows(
return matrix_rows return matrix_rows
def _registry_data_coverage() -> dict: def _registry_data_coverage(
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes() registry_organization_ids: set[str], source_matches: dict[str, set[str]]
) -> dict:
total_organizations = len(registry_organization_ids) total_organizations = len(registry_organization_ids)
descriptor_titles: dict[str, str] = {} descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values(): for descriptor in PARSER_SOURCES.values():
@@ -1998,9 +1999,7 @@ def _registry_data_coverage() -> dict:
for source, _label in ParserLoadLog.Source.choices: for source, _label in ParserLoadLog.Source.choices:
if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES: if source in REGISTRY_COVERAGE_EXCLUDED_SOURCES:
continue continue
organizations_count = len( organizations_count = len(source_matches.get(source, set()))
_matched_source_record_registry_organization_ids(source, by_inn, by_ogrn)
)
coverage_percent = ( coverage_percent = (
round(organizations_count / total_organizations * 100, 1) round(organizations_count / total_organizations * 100, 1)
if total_organizations if total_organizations
@@ -2021,10 +2020,14 @@ def _registry_data_coverage() -> dict:
} }
def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict: def _registry_enrichment_analytics(
registry_organization_ids, by_inn, by_ogrn = _active_registry_organization_indexes() jobs,
schedules: list[dict],
registry_organization_ids: set[str],
source_matches: dict[str, set[str]],
latest_loads: list[ParserLoadLog],
) -> dict:
total_organizations = len(registry_organization_ids) total_organizations = len(registry_organization_ids)
source_matches = _source_registry_matches(by_inn, by_ogrn)
descriptor_titles: dict[str, str] = {} descriptor_titles: dict[str, str] = {}
for descriptor in PARSER_SOURCES.values(): for descriptor in PARSER_SOURCES.values():
descriptor_titles.setdefault(descriptor.source, descriptor.title) descriptor_titles.setdefault(descriptor.source, descriptor.title)
@@ -2113,7 +2116,6 @@ def _registry_enrichment_analytics(jobs, schedules: list[dict]) -> dict:
enrichment_sources=enrichment_sources, enrichment_sources=enrichment_sources,
) )
latest_loads = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
recent_success = sum( recent_success = sum(
1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS 1 for load in latest_loads if load.status == ParserLoadLog.Status.SUCCESS
) )
@@ -2948,6 +2950,9 @@ class ParserDashboardDataView(APIView):
_periodic_task_to_dict(task) _periodic_task_to_dict(task)
for task in _parser_periodic_tasks_for_user(request.user) for task in _parser_periodic_tasks_for_user(request.user)
] ]
load_logs = list(ParserLoadLog.objects.all().order_by("-created_at")[:30])
registry_ids, by_inn, by_ogrn = _active_registry_organization_indexes()
source_matches = _source_registry_matches(by_inn, by_ogrn)
return api_response( return api_response(
{ {
"sources": sources, "sources": sources,
@@ -2960,13 +2965,18 @@ class ParserDashboardDataView(APIView):
"schedules": ParserScheduleSerializer(schedules, many=True).data, "schedules": ParserScheduleSerializer(schedules, many=True).data,
"jobs": BackgroundJobListSerializer(jobs, many=True).data, "jobs": BackgroundJobListSerializer(jobs, many=True).data,
"source_counts": source_counts, "source_counts": source_counts,
"registry_data_coverage": _registry_data_coverage(), "registry_data_coverage": _registry_data_coverage(
registry_ids, source_matches
),
"registry_enrichment_analytics": _registry_enrichment_analytics( "registry_enrichment_analytics": _registry_enrichment_analytics(
jobs, jobs,
schedules, schedules,
registry_ids,
source_matches,
load_logs,
), ),
"load_logs": ParserLoadLogSerializer( "load_logs": ParserLoadLogSerializer(
ParserLoadLog.objects.all().order_by("-created_at")[:30], load_logs,
many=True, many=True,
).data, ).data,
} }

View File

@@ -15,8 +15,18 @@ class Command(BaseAppCommand):
help = "Формирует готовые CSV/XLSX/JSON выгрузки источников" help = "Формирует готовые CSV/XLSX/JSON выгрузки источников"
use_transaction = False use_transaction = False
def add_arguments(self, parser) -> None:
super().add_arguments(parser)
parser.add_argument(
"--all-years",
action="store_true",
help="Все доступные годы в отдельном каталоге all-years",
)
def execute_command(self, *args, **options) -> str: def execute_command(self, *args, **options) -> str:
generation = build_source_record_export_artifacts() generation = build_source_record_export_artifacts(
all_years=options["all_years"]
)
rendered = json.dumps( rendered = json.dumps(
{ {
"generation_id": generation.generation_id, "generation_id": generation.generation_id,

View File

@@ -127,7 +127,7 @@ class SourceRecordExportGeneration:
generation_id: str generation_id: str
generated_at: str generated_at: str
export_year: int export_year: int | None
artifacts: tuple[SourceRecordExportArtifact, ...] artifacts: tuple[SourceRecordExportArtifact, ...]
records_count: int records_count: int
@@ -197,16 +197,19 @@ def build_source_record_export_artifacts(
*, *,
now: datetime | None = None, now: datetime | None = None,
export_directory: str | Path | None = None, export_directory: str | Path | None = None,
all_years: bool = False,
) -> SourceRecordExportGeneration: ) -> SourceRecordExportGeneration:
"""Build all files on disk and atomically publish a new generation.""" """Build annual or all-years files and atomically publish their generation."""
root_directory = _resolve_export_directory(export_directory) root_directory = _resolve_export_directory(export_directory)
if all_years:
root_directory /= "all-years"
generations_directory = root_directory / GENERATION_DIRECTORY_NAME generations_directory = root_directory / GENERATION_DIRECTORY_NAME
root_directory.mkdir(parents=True, exist_ok=True) root_directory.mkdir(parents=True, exist_ok=True)
generations_directory.mkdir(parents=True, exist_ok=True) generations_directory.mkdir(parents=True, exist_ok=True)
generated_at_datetime = _normalize_generation_datetime(now or timezone.now()) generated_at_datetime = _normalize_generation_datetime(now or timezone.now())
export_year = _export_year(generated_at_datetime) export_year = None if all_years else _export_year(generated_at_datetime)
generation_id = ( generation_id = (
f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}" f"{generated_at_datetime.strftime('%Y%m%dT%H%M%SZ')}-{uuid4().hex[:8]}"
) )
@@ -545,7 +548,7 @@ def _build_source_group_file_name(*, source_group: str, file_format: str) -> str
def _source_group_queryset( def _source_group_queryset(
source_group: str, source_group: str,
*, *,
export_year: int, export_year: int | None,
) -> QuerySet[OrganizationSourceRecord]: ) -> QuerySet[OrganizationSourceRecord]:
queryset = ( queryset = (
OrganizationSourceRecord.objects.filter(extension__source_group=source_group) OrganizationSourceRecord.objects.filter(extension__source_group=source_group)
@@ -555,6 +558,8 @@ def _source_group_queryset(
# source groups with millions of rows. # source groups with millions of rows.
.order_by() .order_by()
) )
if export_year is None:
return queryset
if source_group == FINANCIAL_SOURCE_GROUP: if source_group == FINANCIAL_SOURCE_GROUP:
return queryset.filter(financial_lines__year=export_year).distinct() return queryset.filter(financial_lines__year=export_year).distinct()
@@ -581,7 +586,7 @@ def _iter_source_records(
*, *,
source_group: str, source_group: str,
include_financial_lines: bool, include_financial_lines: bool,
export_year: int, export_year: int | None,
) -> Iterator[OrganizationSourceRecord]: ) -> Iterator[OrganizationSourceRecord]:
iterator = _source_group_queryset( iterator = _source_group_queryset(
source_group, source_group,
@@ -597,7 +602,7 @@ def _iter_source_records(
Prefetch( Prefetch(
"financial_lines", "financial_lines",
queryset=OrganizationSourceFinancialLine.objects.filter( queryset=OrganizationSourceFinancialLine.objects.filter(
year=export_year **({"year": export_year} if export_year is not None else {})
).order_by(), ).order_by(),
), ),
) )
@@ -608,7 +613,7 @@ def _spool_source_group_rows(
*, *,
source_group: str, source_group: str,
output_path: Path, output_path: Path,
export_year: int, export_year: int | None,
) -> tuple[list[str], int]: ) -> tuple[list[str], int]:
include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP include_financial_lines = source_group == FINANCIAL_SOURCE_GROUP
payload_headers: set[str] = set() payload_headers: set[str] = set()
@@ -953,6 +958,9 @@ def _generation_manifest_payload(
"generation_id": generation.generation_id, "generation_id": generation.generation_id,
"generated_at": generation.generated_at, "generated_at": generation.generated_at,
"export_year": generation.export_year, "export_year": generation.export_year,
"export_scope": "all_years"
if generation.export_year is None
else "current_year",
"records_count": generation.records_count, "records_count": generation.records_count,
"artifacts_count": generation.artifacts_count, "artifacts_count": generation.artifacts_count,
"files_count": generation.files_count, "files_count": generation.files_count,
@@ -973,6 +981,19 @@ def _generation_manifest_payload(
} }
def _manifest_export_year(payload: dict[str, Any]) -> int | None:
raw_year = payload["export_year"]
scope = payload.get("export_scope", "current_year")
if scope == "all_years" and raw_year is None:
return None
if scope != "current_year" or raw_year is None or isinstance(raw_year, bool):
raise ValueError("Source-record export scope is invalid.")
year = int(raw_year)
if not 1 <= year <= 9999:
raise ValueError("Source-record export year is invalid.")
return year
def _generation_from_manifest( def _generation_from_manifest(
payload: Any, payload: Any,
*, *,
@@ -984,9 +1005,7 @@ def _generation_from_manifest(
generation_id = str(payload["generation_id"]) generation_id = str(payload["generation_id"])
generated_at = str(payload["generated_at"]) generated_at = str(payload["generated_at"])
datetime.fromisoformat(generated_at) datetime.fromisoformat(generated_at)
export_year = int(payload["export_year"]) export_year = _manifest_export_year(payload)
if not 1 <= export_year <= 9999:
raise ValueError("Source-record export year is invalid.")
records_count = int(payload["records_count"]) records_count = int(payload["records_count"])
artifact_payloads = payload["artifacts"] artifact_payloads = payload["artifacts"]
if not isinstance(artifact_payloads, list): if not isinstance(artifact_payloads, list):

View File

@@ -20,6 +20,16 @@ logger = logging.getLogger(__name__)
@shared_task(bind=True, base=CorePeriodicTask) @shared_task(bind=True, base=CorePeriodicTask)
def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001 def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
"""Build and atomically publish the nightly source-record export matrix.""" """Build and atomically publish the nightly source-record export matrix."""
return _refresh_source_record_export_artifacts(self, all_years=False)
@shared_task(bind=True, base=CorePeriodicTask)
def refresh_all_years_source_record_export_artifacts(self) -> dict:
"""Publish all available years independently of the annual generation."""
return _refresh_source_record_export_artifacts(self, all_years=True)
def _refresh_source_record_export_artifacts(task, *, all_years: bool) -> dict:
lock_key = getattr( lock_key = getattr(
settings, settings,
"SOURCE_RECORD_EXPORT_LOCK_KEY", "SOURCE_RECORD_EXPORT_LOCK_KEY",
@@ -29,11 +39,13 @@ def refresh_source_record_export_artifacts(self) -> dict: # noqa: ARG001
getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60) getattr(settings, "SOURCE_RECORD_EXPORT_LOCK_TTL_SECONDS", 6 * 60 * 60)
) )
if not cache.add(lock_key, "1", timeout=lock_ttl): if not cache.add(lock_key, "1", timeout=lock_ttl):
if all_years:
raise task.retry(countdown=60, max_retries=lock_ttl // 60 + 2)
logger.info("Source-record export generation skipped: lock is already held") logger.info("Source-record export generation skipped: lock is already held")
return {"status": "skipped", "reason": "locked"} return {"status": "skipped", "reason": "locked"}
try: try:
generation = build_source_record_export_artifacts() generation = build_source_record_export_artifacts(all_years=all_years)
result = { result = {
"status": "success", "status": "success",
"generation_id": generation.generation_id, "generation_id": generation.generation_id,

View File

@@ -216,6 +216,126 @@ class OrganizationSourceRecordExportApiV2Test(APITestCase):
{export_year}, {export_year},
) )
def test_all_years_generation_is_separate_and_keeps_historical_financial_lines(
self,
):
organization = Organization.objects.create(name="Export", inn="7707083899")
extension = PlannedInspectionExtension.objects.create(organization=organization)
for year in (2020, 2026):
OrganizationSourceRecord.objects.create(
extension=extension,
record_type="inspection",
source="inspections",
external_id=str(year),
record_date=f"{year}-01-01",
)
finance = FinancialIndicatorsExtension.objects.create(organization=organization)
report = OrganizationSourceRecord.objects.create(
extension=finance,
record_type="financial_report",
source="fns_reports",
external_id="full",
)
for year in (2020, 2026):
OrganizationSourceFinancialLine.objects.create(
source_record=report,
year=year,
form_code="1",
line_code=str(year),
)
source = SourceGroup.PLANNED_INSPECTIONS.value
annual = build_source_record_export_artifacts(
now=datetime(2026, 8, 4, tzinfo=UTC)
)
annual_manifest = Path(self.export_directory.name) / "current.json"
before = annual_manifest.read_bytes()
full = build_source_record_export_artifacts(
now=datetime(2026, 8, 4, tzinfo=UTC),
all_years=True,
)
self.assertIsNone(full.export_year)
self.assertEqual(annual_manifest.read_bytes(), before)
full_root = Path(self.export_directory.name) / "all-years"
loaded = load_current_source_record_export_generation(
export_directory=full_root
)
self.assertEqual(loaded.generation_id, full.generation_id)
self.assertIsNone(loaded.export_year)
def rows(generation, group):
artifact = next(
a
for a in generation.artifacts
if a.source_group == group and a.file_format == "json"
)
return _source_records(json.loads(artifact.path.read_text()), group)
self.assertEqual(len(rows(annual, source)), 1)
self.assertEqual(len(rows(full, source)), 2)
self.assertEqual(
{
line["year"]
for line in rows(full, "financial_indicators")[0]["financial_lines"]
},
{2020, 2026},
)
self.assertEqual(
{
line["year"]
for line in rows(annual, "financial_indicators")[0]["financial_lines"]
},
{2026},
)
# Retention in either scope must not delete the other's files.
for _ in range(3):
build_source_record_export_artifacts(all_years=True)
self.assertTrue(all(a.path.is_file() for a in annual.artifacts))
def test_all_years_command_does_not_publish_an_annual_manifest(self):
call_command("build_source_record_exports", all_years=True, stdout=StringIO())
self.assertFalse((Path(self.export_directory.name) / "current.json").exists())
full = load_current_source_record_export_generation(
export_directory=Path(self.export_directory.name) / "all-years",
)
self.assertIsNone(full.export_year)
def test_full_export_task_publishes_separate_manifest_and_waits_for_lock(self):
from celery.exceptions import Retry
from django.core.cache import cache
from organizations.tasks import refresh_all_years_source_record_export_artifacts
lock = "test-full-export-lock"
with self.settings(SOURCE_RECORD_EXPORT_LOCK_KEY=lock):
cache.delete(lock)
cache.add(lock, "held", timeout=30)
try:
with self.assertRaises(Retry):
refresh_all_years_source_record_export_artifacts.run()
self.assertFalse(
(
Path(self.export_directory.name) / "all-years/current.json"
).exists()
)
finally:
cache.delete(lock)
result = refresh_all_years_source_record_export_artifacts.run()
self.assertEqual(result["status"], "success")
self.assertIsNone(result["export_year"])
self.assertIsNone(cache.get(lock))
self.assertFalse(
(Path(self.export_directory.name) / "current.json").exists()
)
def test_manifest_rejects_ambiguous_all_years_scope(self):
build_source_record_export_artifacts(all_years=True)
directory = Path(self.export_directory.name) / "all-years"
manifest_path = directory / "current.json"
payload = json.loads(manifest_path.read_text())
payload["export_year"] = 2026
manifest_path.write_text(json.dumps(payload))
with self.assertRaises(SourceRecordExportArtifactsUnavailable):
load_current_source_record_export_generation(export_directory=directory)
def test_new_calendar_year_requires_a_new_prepared_generation(self): def test_new_calendar_year_requires_a_new_prepared_generation(self):
generated_at = datetime(2026, 12, 31, 23, 59, tzinfo=UTC) generated_at = datetime(2026, 12, 31, 23, 59, tzinfo=UTC)
build_source_record_export_artifacts(now=generated_at) build_source_record_export_artifacts(now=generated_at)

View File

@@ -20,6 +20,8 @@ from apps.parsers.models import (
ProcurementRecord, ProcurementRecord,
) )
from django.core.files.uploadedfile import SimpleUploadedFile from django.core.files.uploadedfile import SimpleUploadedFile
from django.db import connection
from django.test.utils import CaptureQueriesContext
from django.urls import reverse from django.urls import reverse
from openpyxl import Workbook from openpyxl import Workbook
from organizations.models import Organization, OrganizationSourceRecord from organizations.models import Organization, OrganizationSourceRecord
@@ -697,7 +699,17 @@ class ParsersViewSetTest(APITestCase):
) )
self.client.force_authenticate(self.user) self.client.force_authenticate(self.user)
response = self.client.get("/api/v1/parsers/dashboard/") with CaptureQueriesContext(connection) as queries:
response = self.client.get("/api/v1/parsers/dashboard/")
source_scans = [
query["sql"]
for query in queries
if "SELECT DISTINCT" in query["sql"]
and "organizations_source_record" in query["sql"]
]
self.assertTrue(source_scans)
self.assertEqual(len(source_scans), len(set(source_scans)))
self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_200_OK)
coverage = response.data["data"]["registry_data_coverage"] coverage = response.data["data"]["registry_data_coverage"]