fix: publish valid SRO memberships while preserving quarantined history

src/apps/core/serializers.py

@@ -85,6 +85,9 @@ class SnapshotRunMetadataSerializer(serializers.Serializer):
     found_memberships_count = serializers.IntegerField(read_only=True)
     published_records_count = serializers.IntegerField(read_only=True)
     quarantined_records_count = serializers.IntegerField(read_only=True)
+    quarantined_organizations_count = serializers.IntegerField(read_only=True)
+    completed_organizations_count = serializers.IntegerField(read_only=True)
+    retained_records_count = serializers.IntegerField(read_only=True)
     missing_admission_dates_count = serializers.IntegerField(read_only=True)
     http_errors_count = serializers.IntegerField(read_only=True)
     parse_errors_count = serializers.IntegerField(read_only=True)

src/apps/parsers/registry_snapshots.py

@@ -147,22 +147,41 @@ def _invalidate_committed_snapshot_cache() -> None:
         )
 
 
-def _remove_obsolete_records(source, retained_ids, scope_organization_ids) -> None:
+def _remove_obsolete_records(
+    source, retained_ids, scope_organization_ids, preserve_organization_ids
+) -> int:
     obsolete = OrganizationSourceRecord.objects.filter(source=source)
     if scope_organization_ids is not None:
         obsolete = obsolete.filter(
             extension__organization_id__in=scope_organization_ids
         )
+    retained_count = 0
+    if preserve_organization_ids:
+        protected = obsolete.filter(
+            extension__organization_id__in=preserve_organization_ids
+        )
+        retained_count = protected.exclude(external_id__in=retained_ids).count()
+        obsolete = obsolete.exclude(
+            extension__organization_id__in=preserve_organization_ids
+        )
     obsolete.exclude(external_id__in=retained_ids).delete()
+    return retained_count
 
 
-def _set_published_count(artifact, source, published, scope_organization_ids) -> None:
+def _set_published_count(
+    artifact, source, published, scope_organization_ids, retained_count=None
+) -> None:
     artifact.published_count = published
-    if scope_organization_ids is not None:
+    if scope_organization_ids is not None or retained_count is not None:
         artifact.metadata = {**artifact.metadata, "updated_records_count": published}
         artifact.published_count = OrganizationSourceRecord.objects.filter(
             source=source
         ).count()
+    if retained_count is not None:
+        artifact.metadata = {
+            **artifact.metadata,
+            "retained_records_count": retained_count,
+        }
     organizations_count = (
         OrganizationSourceRecord.objects.filter(source=source)
         .order_by()
@@ -183,8 +202,13 @@ def publish_snapshot(
     *,
     on_publish: Callable[[ParserSourceArtifact, SnapshotResult], None] | None = None,
     scope_organization_ids: list[uuid.UUID] | None = None,
+    preserve_organization_ids: list[uuid.UUID] | None = None,
 ) -> SnapshotResult:
-    """Publish complete staging in one transaction, preserving stable record IDs."""
+    """Publish validated staging atomically; optionally retain uncertain old rows.
+
+    Preservation affects only obsolete deletion for the selected organizations.
+    Valid staged rows are still upserted, including their existing stable IDs.
+    """
     source = artifact.source
     descriptor = get_source_group_descriptor(source)
     staged = ParserStagedRecord.objects.filter(
@@ -253,13 +277,21 @@ def publish_snapshot(
                 raise SnapshotValidationError("organization_resolution_changed")
             published += len(inputs)
         retained_ids = staged.values_list("external_id", flat=True)
-        _remove_obsolete_records(source, retained_ids, scope_organization_ids)
+        retained_count = _remove_obsolete_records(
+            source, retained_ids, scope_organization_ids, preserve_organization_ids
+        )
         descriptor.extension_model.objects.filter(records__isnull=True).delete()
         # Deletion can change counts for extensions whose remaining rows were upserted
         # before old records were removed.
         _refresh_extension_counts(descriptor)
         staged.update(disposition=ParserStagedRecord.Disposition.PUBLISHED)
-        _set_published_count(artifact, source, published, scope_organization_ids)
+        _set_published_count(
+            artifact,
+            source,
+            published,
+            scope_organization_ids,
+            retained_count if preserve_organization_ids is not None else None,
+        )
         artifact.status = ParserSourceArtifact.Status.PUBLISHED
         artifact.save()
         result = _artifact_result(artifact)

src/apps/parsers/sro_membership.py

@@ -447,6 +447,7 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
     raw_count, not_found, missing_dates = 0, 0, 0
     parse_errors = 0
     recognized_lookup_404, successful_lookup_200 = 0, 0
+    confirmed_empty_lookup_200 = 0
     reasons, dates = Counter(), set()
 
     def fetch(url):
@@ -480,6 +481,7 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
             rows, version = parse_sro_lookup(page, expected_lookup_value=value)
             recognized_lookup_404 += page.status_code == 404
             successful_lookup_200 += page.status_code == 200
+            confirmed_empty_lookup_200 += page.status_code == 200 and not rows
             if version:
                 dates.add(version)
             if len(dates) > 1:
@@ -547,6 +549,7 @@ def _collect_sro_candidates(artifact, candidates, http, raw, on_progress):
             not_found_organizations_count=not_found,
             recognized_lookup_404_count=recognized_lookup_404,
             successful_lookup_200_count=successful_lookup_200,
+            confirmed_empty_lookup_200_count=confirmed_empty_lookup_200,
             raw_memberships_count=raw_count,
             found_memberships_count=raw_count,
             quarantined_records_count=sum(reasons.values()),
@@ -609,6 +612,34 @@ def _save_sro_checkpoints(candidates, artifact) -> None:
         )
 
 
+def _sro_publication_scope(artifact, candidates):
+    """Keep uncertain old memberships without discarding independently valid rows."""
+    staged = artifact.staged_records.all()
+    if staged.filter(organization_id__isnull=True).exists():
+        raise SnapshotValidationError("organization_resolution_changed")
+    quarantined_ids = set(
+        staged.filter(disposition=ParserStagedRecord.Disposition.QUARANTINED)
+        .order_by()
+        .values_list("organization_id", flat=True)
+        .distinct()
+    )
+    completed = [org for org in candidates if org.uid not in quarantined_ids]
+    artifact.metadata.update(
+        quarantined_organizations_count=len(quarantined_ids),
+        completed_organizations_count=len(completed),
+    )
+    if (
+        quarantined_ids
+        and not staged.filter(
+            disposition=ParserStagedRecord.Disposition.STAGED
+        ).exists()
+        and not artifact.metadata["confirmed_empty_lookup_200_count"]
+    ):
+        # Invalid rows plus an ambiguous 404 do not establish any valid coverage.
+        raise SnapshotValidationError("sro_incomplete_membership_scan")
+    return completed, list(quarantined_ids)
+
+
 def refresh_sro_membership(
     *,
     load_batch: int,
@@ -641,11 +672,6 @@ def refresh_sro_membership(
                     )
             finally:
                 save_artifact_file(artifact, handle)
-        if artifact.quarantined_count:
-            # A semantic failure is not evidence that old memberships disappeared.
-            # Keep raw/quarantine diagnostics, but never advance any checkpoint or
-            # replace the previous complete dataset with this incomplete scan.
-            raise SnapshotValidationError("sro_incomplete_membership_scan")
         if (
             candidates
             and artifact.metadata["recognized_lookup_404_count"] == len(candidates)
@@ -654,6 +680,7 @@ def refresh_sro_membership(
             # An all-404 scan cannot distinguish absence from a site-wide outage.
             # Raw responses stay available, but previous rows/checkpoints survive.
             raise SnapshotValidationError("sro_ambiguous_empty_scan")
+        completed, quarantined_ids = _sro_publication_scope(artifact, candidates)
         if mode == "incremental" and not candidates:
             previous = (
                 ParserSourceArtifact.objects.filter(
@@ -675,7 +702,12 @@ def refresh_sro_membership(
         artifact.save()
 
         def finalize(published_artifact, result):
-            _save_sro_checkpoints(candidates, published_artifact)
+            # An unchanged fingerprint must not suppress retry after a full sweep
+            # found invalid rows. Deletion shares the publication transaction.
+            SroOrganizationLookup.objects.filter(
+                organization_id__in=quarantined_ids
+            ).delete()
+            _save_sro_checkpoints(completed, published_artifact)
             if on_publish:
                 on_publish(published_artifact, result)
 
@@ -684,6 +716,7 @@ def refresh_sro_membership(
             scope_organization_ids=[org.uid for org in candidates]
             if mode == "incremental"
             else None,
+            preserve_organization_ids=quarantined_ids,
             on_publish=finalize,
         )
     except Exception as exc: