slide2vec 4.6.3__tar.gz → 4.7.0__tar.gz

{slide2vec-4.6.3 → slide2vec-4.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: slide2vec
-Version: 4.6.3
+Version: 4.7.0
 Summary: Embedding of whole slide images with Foundation Models
 Author-email: Clément Grisi <clement.grisi@radboudumc.nl>
 License-Expression: Apache-2.0

{slide2vec-4.6.3 → slide2vec-4.7.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "slide2vec"
-version = "4.6.3"
+version = "4.7.0"
 description = "Embedding of whole slide images with Foundation Models"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -164,7 +164,7 @@ no_implicit_reexport = true
 max-line-length = 160
 
 [tool.bumpver]
-current_version = "4.6.3"
+current_version = "4.7.0"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit = false       # We do version bumping in CI, not as a commit
 tag = false          # Git tag already exists — we don't auto-tag

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/__init__.py

@@ -11,7 +11,7 @@ from slide2vec.api import (
 from slide2vec.artifacts import HierarchicalEmbeddingArtifact, SlideEmbeddingArtifact, TileEmbeddingArtifact
 
 
-__version__ = "4.6.3"
+__version__ = "4.7.0"
 
 __all__ = [
     "Model",

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/encoders/models/conch.py

@@ -141,6 +141,13 @@ class CONCH(TileEncoder):
     def encode_dim(self) -> int:
         return 512
 
+    @property
+    def patch_size(self) -> tuple[int, int]:
+        # The CONCH vision trunk is a timm ViT-B/16; expose its patch size so the
+        # dense path can resolve the token grid. open_clip builds the trunk without
+        # dynamic_img_size, so dense extraction must use the native 448 window.
+        return _patch_size_from_trunk(self._model.visual.trunk)
+
     @property
     def device(self) -> torch.device:
         return self._device
@@ -204,6 +211,11 @@ class CONCHv15(TileEncoder):
     def encode_dim(self) -> int:
         return 768
 
+    @property
+    def patch_size(self) -> tuple[int, int]:
+        # CONCHv15's TITAN trunk is a timm ViT; expose its patch size for the dense path.
+        return _patch_size_from_trunk(self._model.trunk)
+
     @property
     def device(self) -> torch.device:
         return self._device

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/encoders/models/hibou.py

@@ -123,6 +123,12 @@ class _HibouBase(TileEncoder):
     def encode_dim(self) -> int:
         return self._encode_dim
 
+    @property
+    def patch_size(self) -> tuple[int, int]:
+        # HF Dinov2-style config carries the patch size; expose it for the dense path.
+        patch = int(self._model.config.patch_size)
+        return patch, patch
+
     @property
     def device(self) -> torch.device:
         return self._device

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/encoders/models/midnight.py

@@ -124,6 +124,12 @@ class Midnight(TileEncoder):
     def encode_dim(self) -> int:
         return 3072
 
+    @property
+    def patch_size(self) -> tuple[int, int]:
+        # HF Dinov2-style config carries the patch size; expose it for the dense path.
+        patch = int(self._model.config.patch_size)
+        return patch, patch
+
     @property
     def device(self) -> torch.device:
         return self._device

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/encoders/models/musk.py

@@ -114,6 +114,11 @@ class MUSK(TileEncoder):
     def encode_dim(self) -> int:
         return 2048 if self._output_variant == "ms_aug" else 1024  # cls
 
+    @property
+    def patch_size(self) -> tuple[int, int]:
+        # BEiT3 vision embedding carries the patch size; expose it for the dense path.
+        return _as_hw(self._model.beit3.vision_embed.patch_size)
+
     @property
     def device(self) -> torch.device:
         return self._device

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/inference.py

@@ -148,6 +148,52 @@ def load_model(
     )
 
 
+def _reconcile_embedding_process_list(
+    *,
+    model,
+    preprocessing: PreprocessingConfig,
+    execution: ExecutionOptions,
+    process_list_path,
+    embeddable_slides,
+    output_dir,
+):
+    """Reconcile the process_list with the embeddings on disk once, at end of run.
+
+    The incremental persist callback batches its process_list writes for the tile
+    path, so the trailing partial batch is only persisted by this final full-CSV
+    reconciliation. Every single-GPU embedding entry point must call it after its
+    embed loop. Collecting artifacts from disk (rather than the callback's
+    in-memory list) also covers resume-skipped slides. Returns the collected
+    (tile, hierarchical, slide) artifact lists.
+    """
+    persist_tile_embeddings = embedding.should_persist_tile_embeddings(model, execution)
+    persist_hierarchical_embeddings = hierarchical.is_hierarchical_preprocessing(preprocessing)
+    include_slide_embeddings = model.level == "slide"
+    include_tile_embeddings = persist_tile_embeddings and not persist_hierarchical_embeddings
+    tile_artifacts, hierarchical_artifacts, slide_artifacts = artifacts_collect.collect_pipeline_artifacts(
+        embeddable_slides,
+        output_dir=output_dir,
+        output_format=execution.output_format,
+        include_tile_embeddings=include_tile_embeddings,
+        include_hierarchical_embeddings=persist_hierarchical_embeddings,
+        include_slide_embeddings=include_slide_embeddings,
+    )
+    if process_list_path is not None and Path(process_list_path).is_file():
+        persistence.update_process_list_after_embedding(
+            process_list_path,
+            successful_slides=embeddable_slides,
+            persist_tile_embeddings=persist_tile_embeddings,
+            persist_hierarchical_embeddings=persist_hierarchical_embeddings,
+            include_slide_embeddings=include_slide_embeddings,
+            encoder_name=model.name,
+            output_variant=process_list.resolved_process_list_output_variant(model),
+            tile_artifacts=tile_artifacts,
+            hierarchical_artifacts=hierarchical_artifacts,
+            slide_artifacts=slide_artifacts,
+        )
+    return tile_artifacts, hierarchical_artifacts, slide_artifacts
+
+
 def embed_slides(
     model,
     slides,
@@ -266,6 +312,18 @@ def embed_slides(
                         hierarchical_artifacts=hierarchical_artifacts,
                         slide_artifacts=slide_artifacts,
                     )
+            elif execution.output_dir is not None:
+                # Single-GPU: the incremental callback persisted the embeddings but
+                # batches its process_list writes, so reconcile the full CSV once at
+                # the end (covers the trailing partial batch on a clean run).
+                _reconcile_embedding_process_list(
+                    model=model,
+                    preprocessing=preprocessing,
+                    execution=execution,
+                    process_list_path=process_list_path,
+                    embeddable_slides=embeddable_slides,
+                    output_dir=Path(execution.output_dir),
+                )
             emit_progress(
                 "embedding.finished",
                 slide_count=len(embeddable_slides),
@@ -752,7 +810,6 @@ def run_pipeline(
         persist_tile_embeddings = embedding.should_persist_tile_embeddings(model, execution)
         persist_hierarchical_embeddings = hierarchical.is_hierarchical_preprocessing(resolved_preprocessing)
         include_slide_embeddings = model.level == "slide"
-        include_tile_embeddings = persist_tile_embeddings and not persist_hierarchical_embeddings
         pending_slides, pending_tiling_results = persist_callbacks.pending_local_embedding_records(
             embeddable_slides,
             embeddable_tiling_results,
@@ -790,25 +847,13 @@ def run_pipeline(
                 on_embedded_slide=local_persist_callback,
                 collect_results=False,
             )
-        tile_artifacts, hierarchical_artifacts, slide_artifacts = artifacts_collect.collect_pipeline_artifacts(
-            embeddable_slides,
+        tile_artifacts, hierarchical_artifacts, slide_artifacts = _reconcile_embedding_process_list(
+            model=model,
+            preprocessing=resolved_preprocessing,
+            execution=execution,
+            process_list_path=process_list_path,
+            embeddable_slides=embeddable_slides,
             output_dir=output_dir,
-            output_format=execution.output_format,
-            include_tile_embeddings=include_tile_embeddings,
-            include_hierarchical_embeddings=persist_hierarchical_embeddings,
-            include_slide_embeddings=include_slide_embeddings,
-        )
-        persistence.update_process_list_after_embedding(
-            process_list_path,
-            successful_slides=embeddable_slides,
-            persist_tile_embeddings=persist_tile_embeddings,
-            persist_hierarchical_embeddings=persist_hierarchical_embeddings,
-            include_slide_embeddings=include_slide_embeddings,
-            encoder_name=model.name,
-            output_variant=process_list.resolved_process_list_output_variant(model),
-            tile_artifacts=tile_artifacts,
-            hierarchical_artifacts=hierarchical_artifacts,
-            slide_artifacts=slide_artifacts,
         )
         emit_progress(
             "embedding.finished",
@@ -907,7 +952,7 @@ def run_pipeline_with_coordinates(
                 slide_artifacts=slide_artifacts,
                 process_list_path=process_list_path,
             )
-        local_persist_callback, tile_or_hier_artifacts, slide_artifacts = persist_callbacks.build_incremental_persist_callback(
+        local_persist_callback, _, _ = persist_callbacks.build_incremental_persist_callback(
             model=model,
             preprocessing=resolved_preprocessing,
             execution=execution,
@@ -922,17 +967,18 @@ def run_pipeline_with_coordinates(
             on_embedded_slide=local_persist_callback,
             collect_results=False,
         )
-        tile_artifacts: list[TileEmbeddingArtifact] = []
-        hierarchical_artifacts: list[HierarchicalEmbeddingArtifact] = []
-        for artifact in tile_or_hier_artifacts:
-            if isinstance(artifact, HierarchicalEmbeddingArtifact):
-                hierarchical_artifacts.append(artifact)
-            elif artifact is not None:
-                tile_artifacts.append(artifact)
+        tile_artifacts, hierarchical_artifacts, slide_artifacts = _reconcile_embedding_process_list(
+            model=model,
+            preprocessing=resolved_preprocessing,
+            execution=execution,
+            process_list_path=process_list_path,
+            embeddable_slides=embeddable_slides,
+            output_dir=output_dir,
+        )
         return RunResult(
             tile_artifacts=tile_artifacts,
             hierarchical_artifacts=hierarchical_artifacts,
-            slide_artifacts=list(slide_artifacts),
+            slide_artifacts=slide_artifacts,
             process_list_path=process_list_path,
         )
     except Exception as exc:

slide2vec-4.7.0/slide2vec/runtime/dense_regions.py

@@ -0,0 +1,229 @@
+"""Dense ``(d, h, w)`` grid extraction over **slide regions at coordinates**.
+
+The dense counterpart of the pooled coordinate path (``compute_tile_embeddings_for_slide``
+→ ``run_forward_pass`` → ``encode_tiles``): instead of pooling each region to one vector,
+each sampled ROI is read **spacing-aware** from the slide, run through the encoder's
+normalization-only dense transform (``get_dense_transform`` — NOT the pooled transform,
+which crops), padded up to the encoder's patch multiple, and encoded via
+``encode_tiles_dense`` into a ``(d, grid_h, grid_w)`` token grid.
+
+This is the extraction half of soma's slide-manifest segmentation path: slide2vec reads
+regions + encodes (it already owns the region reader and the dense encode); soma sources
+the ROI coordinates (hs2p annotation sampling) and persists/caches the grids. It mirrors
+the pooled split exactly — extraction here, caching in soma.
+
+Region reads are spacing-aware via hs2p (:meth:`hs2p.wsi.wsi.WSI.read_region_at_spacing`):
+the finest pyramid level ``<=`` the requested µm/px is read and downscaled to the exact
+``target_size`` (``area`` for images), so the token grid registers against a mask read at
+the same spacing. The ``wsi`` is injected (any object exposing ``read_region_at_spacing``),
+so the loop is unit-testable offline with a fake reader + a random-weight encoder.
+
+Whole-tile only (one padded forward per region). Sliding-window dense extraction over
+coordinates (``window_size`` < input) is a deferred follow-up — large ROIs that exceed the
+encoder's comfortable field are out of scope for the first increment.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Sequence
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+
+from slide2vec.runtime.slide_encode import slide_encode_autocast_ctx
+
+_PAD_MODES = {"reflect", "constant", "zero", "replicate"}
+
+
+def _normalize_hw(value: int | tuple[int, int], *, name: str) -> tuple[int, int]:
+    if isinstance(value, int):
+        if value <= 0:
+            raise ValueError(f"{name} must be positive, got {value}")
+        return value, value
+    try:
+        h, w = value
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"{name} must be an int or an (h, w) pair, got {value!r}") from exc
+    h, w = int(h), int(w)
+    if h <= 0 or w <= 0:
+        raise ValueError(f"{name} must be positive, got {(h, w)}")
+    return h, w
+
+
+def _round_up(value: int, multiple: int) -> int:
+    return ((value + multiple - 1) // multiple) * multiple
+
+
+@dataclass(frozen=True)
+class DenseGridGeometry:
+    """Resolved spatial layout for one dense extraction (slide2vec-owned).
+
+    ``target_size`` is the supervision tile size (h, w); ``encoded_size`` is that rounded
+    up to the patch multiple (pad on bottom/right); ``grid_shape`` is the resulting token
+    grid (grid_h, grid_w). Mirrors soma's ``DenseGridGeometry`` — the dense-grid geometry
+    is extraction geometry and belongs in the extraction engine; soma reads it back from
+    the persisted sidecar.
+    """
+
+    target_size: tuple[int, int]
+    patch_size: tuple[int, int]
+    encoded_size: tuple[int, int]
+    grid_shape: tuple[int, int]
+    pad: tuple[int, int]  # (pad_bottom, pad_right)
+
+
+def compute_dense_geometry(
+    *, target_size: int | tuple[int, int], patch_size: int | tuple[int, int]
+) -> DenseGridGeometry:
+    """Encoded size, token grid, and bottom/right padding for a ``target_size`` tile."""
+    target_h, target_w = _normalize_hw(target_size, name="target_size")
+    patch_h, patch_w = _normalize_hw(patch_size, name="patch_size")
+    encoded_h = _round_up(target_h, patch_h)
+    encoded_w = _round_up(target_w, patch_w)
+    return DenseGridGeometry(
+        target_size=(target_h, target_w),
+        patch_size=(patch_h, patch_w),
+        encoded_size=(encoded_h, encoded_w),
+        grid_shape=(encoded_h // patch_h, encoded_w // patch_w),
+        pad=(encoded_h - target_h, encoded_w - target_w),
+    )
+
+
+def pad_image_to_encoded(
+    tensor: torch.Tensor,
+    geometry: DenseGridGeometry,
+    *,
+    pad_mode: str,
+    image_pad_value: float | None,
+) -> torch.Tensor:
+    """Pad a ``(C, H, W)`` tile (bottom/right) up to ``geometry.encoded_size``."""
+    pad_bottom, pad_right = geometry.pad
+    if pad_bottom == 0 and pad_right == 0:
+        return tensor
+    x = tensor.unsqueeze(0)  # F.pad's 2-D modes need a batch dim
+    pad = (0, pad_right, 0, pad_bottom)  # (left, right, top, bottom)
+    if pad_mode in ("constant", "zero"):
+        x = F.pad(x, pad, mode="constant", value=float(image_pad_value or 0.0))
+    else:
+        x = F.pad(x, pad, mode=pad_mode)
+    return x.squeeze(0)
+
+
+def _resolve_encode_fn(
+    model,
+    *,
+    feature_kind: str,
+    attention_blocks: tuple[int, ...],
+    attention_include_registers: bool,
+) -> Callable[[torch.Tensor], torch.Tensor]:
+    if feature_kind == "patch_features":
+        return model.encode_tiles_dense
+    if feature_kind == "cls_attention":
+        blocks = tuple(int(b) for b in attention_blocks)
+        include_registers = bool(attention_include_registers)
+
+        def encode_fn(window: torch.Tensor) -> torch.Tensor:
+            return model.encode_tiles_attention(
+                window, blocks=blocks, include_registers=include_registers
+            )
+
+        return encode_fn
+    raise ValueError(
+        f"unsupported feature_kind {feature_kind!r}; expected 'patch_features' or 'cls_attention'"
+    )
+
+
+def encode_regions_dense(
+    *,
+    model,
+    device: torch.device | str,
+    wsi,
+    coordinates: Sequence[tuple[int, int]],
+    requested_spacing_um: float,
+    target_size: int | tuple[int, int],
+    tolerance: float = 0.05,
+    pad_mode: str = "reflect",
+    image_pad_value: float | None = None,
+    feature_kind: str = "patch_features",
+    attention_blocks: tuple[int, ...] = (-1,),
+    attention_include_registers: bool = False,
+    batch_size: int = 1,
+    precision: str = "fp32",
+    dense_transform: Callable | None = None,
+) -> np.ndarray:
+    """Encode slide regions at ``coordinates`` into dense grids; return ``(N, d, gh, gw)``.
+
+    Injectable core: takes a constructed dense-capable ``model`` (with
+    ``encode_tiles_dense`` / ``encode_tiles_attention`` / ``patch_size`` /
+    ``get_dense_transform``) and a ``wsi`` exposing
+    ``read_region_at_spacing(location, requested_spacing_um, size, *, tolerance,
+    interpolation)``, so it runs offline in tests with random weights + a fake reader.
+
+    Args:
+        coordinates: ``(x, y)`` top-left locations in **level-0** pixel space (the hs2p
+            tiling convention; passed straight to ``read_region_at_spacing``).
+        requested_spacing_um: µm/px to read each region at.
+        target_size: supervision tile size (int or ``(h, w)``); the region is read at this
+            size at ``requested_spacing_um`` and the token grid registers to it.
+
+    Returns a ``float32`` array of dense grids in coordinate order. ``feature_kind``
+    selects ``encode_tiles_dense`` (patch grid) vs ``encode_tiles_attention`` (CLS-attention
+    grid); both produce a ``(C, gh, gw)`` grid and share this path.
+    """
+    if pad_mode not in _PAD_MODES:
+        raise ValueError(f"unsupported pad_mode {pad_mode!r}; expected one of {sorted(_PAD_MODES)}")
+    geometry = compute_dense_geometry(target_size=target_size, patch_size=model.patch_size)
+    if dense_transform is None:
+        dense_transform = model.get_dense_transform()
+    encode_fn = _resolve_encode_fn(
+        model,
+        feature_kind=feature_kind,
+        attention_blocks=attention_blocks,
+        attention_include_registers=attention_include_registers,
+    )
+    target_h, target_w = geometry.target_size
+
+    coords = [(int(x), int(y)) for x, y in coordinates]
+    grid_h, grid_w = geometry.grid_shape
+    if not coords:
+        return np.empty((0, 0, grid_h, grid_w), dtype=np.float32)
+
+    def _read_padded(location: tuple[int, int]) -> torch.Tensor:
+        region = wsi.read_region_at_spacing(
+            location,
+            float(requested_spacing_um),
+            (target_w, target_h),  # hs2p size is (width, height)
+            tolerance=float(tolerance),
+            interpolation="area",
+        )
+        region = np.ascontiguousarray(np.asarray(region)[..., :3])
+        tensor = torch.as_tensor(dense_transform(Image.fromarray(region))).as_subclass(torch.Tensor)
+        if tensor.ndim != 3:
+            raise ValueError(
+                f"dense transform at {location} produced a {tensor.ndim}-D tensor; expected (C, H, W)."
+            )
+        if tuple(int(s) for s in tensor.shape[-2:]) != (target_h, target_w):
+            raise ValueError(
+                f"region at {location} is {tuple(int(s) for s in tensor.shape[-2:])} after the dense "
+                f"transform, but target_size is {(target_h, target_w)}. The dense transform must be "
+                "normalization-only (no resize/crop)."
+            )
+        return pad_image_to_encoded(
+            tensor, geometry, pad_mode=pad_mode, image_pad_value=image_pad_value
+        )
+
+    grids: list[np.ndarray] = []
+    with torch.inference_mode(), slide_encode_autocast_ctx(device, precision):
+        for start in range(0, len(coords), max(1, int(batch_size))):
+            chunk = coords[start : start + max(1, int(batch_size))]
+            batch = torch.stack([_read_padded(loc) for loc in chunk]).to(device, non_blocking=True)
+            out = encode_fn(batch)
+            if out.ndim != 4:
+                raise ValueError(
+                    f"{feature_kind} encode returned a {out.ndim}-D tensor; expected (B, d, gh, gw)."
+                )
+            grids.append(out.detach().float().cpu().numpy())
+    return np.concatenate(grids, axis=0)

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec/runtime/persist_callbacks.py

@@ -18,6 +18,16 @@ from slide2vec.runtime.persistence import update_process_list_after_embedding
 from slide2vec.runtime.process_list import resolved_process_list_output_variant
 from slide2vec.utils.tiling_io import load_embedding_process_df
 
+# Number of completed tile-level samples to buffer before rewriting the
+# process_list CSV. Each rewrite re-reads and re-writes the *entire* CSV, so
+# doing it once per sample is O(N^2) in I/O when every tile is its own sample
+# (e.g. patch-level benchmarks with hundreds of thousands of tiles). Batching
+# makes it O(N) while only risking the re-embedding of at most this many cheap
+# tile samples after a crash (a clean run reconciles the full CSV at the end).
+# Slide- and hierarchical-level runs (sample == slide: few, expensive samples)
+# keep a flush interval of 1 so every completed slide is checkpointed.
+TILE_EMBEDDING_FLUSH_INTERVAL = 1000
+
 
 def has_complete_local_embedding_outputs(
     sample_id: str,
@@ -141,6 +151,45 @@ def build_incremental_persist_callback(
     persist_hierarchical_embeddings = is_hierarchical_preprocessing(preprocessing)
     include_slide_embeddings = model.level == "slide"
 
+    # Only the pure tile-level path produces the many-cheap-samples workload that
+    # makes per-sample CSV rewrites O(N^2). When the model aggregates to slide
+    # level (or runs hierarchically) the sample is a slide/region — few and
+    # expensive — so checkpoint every one (interval 1). save_tile_embeddings on a
+    # slide-level model still iterates per slide, hence the include_slide check.
+    is_tile_level = (
+        persist_tile_embeddings
+        and not persist_hierarchical_embeddings
+        and not include_slide_embeddings
+    )
+    flush_interval = TILE_EMBEDDING_FLUSH_INTERVAL if is_tile_level else 1
+
+    # Buffered completions awaiting the next batched process_list rewrite.
+    pending_slides: list[SlideSpec] = []
+    pending_tile_artifacts: list[TileEmbeddingArtifact] = []
+    pending_hierarchical_artifacts: list[HierarchicalEmbeddingArtifact] = []
+    pending_slide_artifacts: list[SlideEmbeddingArtifact] = []
+
+    def _flush_process_list() -> None:
+        if not pending_slides:
+            return
+        if process_list_path is not None and process_list_path.is_file():
+            update_process_list_after_embedding(
+                process_list_path,
+                successful_slides=list(pending_slides),
+                persist_tile_embeddings=persist_tile_embeddings,
+                persist_hierarchical_embeddings=persist_hierarchical_embeddings,
+                include_slide_embeddings=include_slide_embeddings,
+                encoder_name=model.name,
+                output_variant=resolved_process_list_output_variant(model),
+                tile_artifacts=list(pending_tile_artifacts),
+                hierarchical_artifacts=list(pending_hierarchical_artifacts),
+                slide_artifacts=list(pending_slide_artifacts),
+            )
+        pending_slides.clear()
+        pending_tile_artifacts.clear()
+        pending_hierarchical_artifacts.clear()
+        pending_slide_artifacts.clear()
+
     def _persist_completed_slide(slide: SlideSpec, tiling_result, embedded_slide: EmbeddedSlide) -> None:
         tile_artifact, slide_artifact = persist_embedded_slide(
             model,
@@ -153,18 +202,16 @@ def build_incremental_persist_callback(
             tile_artifacts.append(tile_artifact)
         if slide_artifact is not None:
             slide_artifacts.append(slide_artifact)
-        if process_list_path is not None and process_list_path.is_file():
-            update_process_list_after_embedding(
-                process_list_path,
-                successful_slides=[slide],
-                persist_tile_embeddings=persist_tile_embeddings,
-                persist_hierarchical_embeddings=persist_hierarchical_embeddings,
-                include_slide_embeddings=include_slide_embeddings,
-                encoder_name=model.name,
-                output_variant=resolved_process_list_output_variant(model),
-                tile_artifacts=[tile_artifact] if isinstance(tile_artifact, TileEmbeddingArtifact) else [],
-                hierarchical_artifacts=[tile_artifact] if isinstance(tile_artifact, HierarchicalEmbeddingArtifact) else [],
-                slide_artifacts=[slide_artifact] if slide_artifact is not None else [],
-            )
+        # Buffer this completion; a slide with no successful artifact is still
+        # recorded so the batched rewrite can mark its feature_status="error".
+        pending_slides.append(slide)
+        if isinstance(tile_artifact, TileEmbeddingArtifact):
+            pending_tile_artifacts.append(tile_artifact)
+        elif isinstance(tile_artifact, HierarchicalEmbeddingArtifact):
+            pending_hierarchical_artifacts.append(tile_artifact)
+        if slide_artifact is not None:
+            pending_slide_artifacts.append(slide_artifact)
+        if len(pending_slides) >= flush_interval:
+            _flush_process_list()
 
     return _persist_completed_slide, tile_artifacts, slide_artifacts

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: slide2vec
-Version: 4.6.3
+Version: 4.7.0
 Summary: Embedding of whole slide images with Foundation Models
 Author-email: Clément Grisi <clement.grisi@radboudumc.nl>
 License-Expression: Apache-2.0

{slide2vec-4.6.3 → slide2vec-4.7.0}/slide2vec.egg-info/SOURCES.txt

@@ -53,6 +53,7 @@ slide2vec/runtime/__init__.py
 slide2vec/runtime/artifacts_collect.py
 slide2vec/runtime/batching.py
 slide2vec/runtime/cpu_budget.py
+slide2vec/runtime/dense_regions.py
 slide2vec/runtime/distributed.py
 slide2vec/runtime/distributed_stage.py
 slide2vec/runtime/embedding.py
@@ -83,6 +84,7 @@ tests/test_architecture_runtime_split.py
 tests/test_attention_extraction.py
 tests/test_dense_extraction.py
 tests/test_dense_locality_gated.py
+tests/test_dense_regions.py
 tests/test_encoder_registry.py
 tests/test_hs2p_package_cutover.py
 tests/test_output_consistency.py

slide2vec-4.7.0/tests/test_dense_regions.py

@@ -0,0 +1,117 @@
+"""Tests for dense grid extraction over slide regions: ``encode_regions_dense``.
+
+Fully offline (``pretrained=False`` random weights) + an injected fake reader, so no
+weights, no real WSI. Checks (1) grid shapes over a batch of coordinates and (2) that the
+orchestration is a faithful wrapper — its per-region grid is byte-identical to a direct
+``encode_tiles_dense(transform → pad)`` of the same region.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+torch = pytest.importorskip("torch")
+timm = pytest.importorskip("timm")
+
+from slide2vec.encoders.base import TimmTileEncoder  # noqa: E402
+from slide2vec.runtime.dense_regions import (  # noqa: E402
+    compute_dense_geometry,
+    encode_regions_dense,
+    pad_image_to_encoded,
+)
+
+
+def _encoder(**kwargs) -> TimmTileEncoder:
+    return TimmTileEncoder("vit_tiny_patch16_224", pretrained=False, num_classes=0,
+                           dynamic_img_size=True, **kwargs)
+
+
+class _FakeWSI:
+    """Returns a deterministic RGB region per location (so reads are reproducible)."""
+
+    def __init__(self, *, target_h: int, target_w: int):
+        self._target_h = target_h
+        self._target_w = target_w
+        self.calls: list[tuple] = []
+
+    def read_region_at_spacing(self, location, requested_spacing_um, size, *, tolerance, interpolation):
+        self.calls.append((tuple(location), requested_spacing_um, tuple(size), tolerance, interpolation))
+        width, height = size
+        x, y = location
+        rng = np.random.default_rng(abs(hash((int(x), int(y)))) % (2**32))
+        return rng.integers(0, 256, size=(height, width, 3), dtype=np.uint8)
+
+
+def test_encode_regions_dense_shapes_over_coordinates():
+    enc = _encoder()
+    target_size = 64  # patch 16 -> grid 4x4, no padding
+    wsi = _FakeWSI(target_h=target_size, target_w=target_size)
+    coords = [(0, 0), (64, 0), (0, 64)]
+
+    grids = encode_regions_dense(
+        model=enc,
+        device="cpu",
+        wsi=wsi,
+        coordinates=coords,
+        requested_spacing_um=0.5,
+        target_size=target_size,
+        batch_size=2,
+    )
+
+    assert grids.shape == (3, enc.encode_dim, 4, 4)
+    assert grids.dtype == np.float32
+    # Reads went through read_region_at_spacing at (target_w, target_h), area interp, level-0 coords.
+    assert [c[0] for c in wsi.calls] == [(0, 0), (64, 0), (0, 64)]
+    assert all(c[2] == (target_size, target_size) and c[4] == "area" for c in wsi.calls)
+
+
+def test_encode_regions_dense_pads_non_multiple_target():
+    enc = _encoder()
+    target_size = 60  # padded up to 64 -> grid 4x4
+    wsi = _FakeWSI(target_h=target_size, target_w=target_size)
+    grids = encode_regions_dense(
+        model=enc, device="cpu", wsi=wsi, coordinates=[(0, 0)],
+        requested_spacing_um=0.5, target_size=target_size,
+    )
+    assert grids.shape == (1, enc.encode_dim, 4, 4)
+
+
+def test_encode_regions_dense_matches_direct_encode():
+    """The primitive is a faithful wrapper: parity vs a hand-rolled transform+pad+encode."""
+    enc = _encoder()
+    target_size = 64
+    wsi = _FakeWSI(target_h=target_size, target_w=target_size)
+    coords = [(0, 0), (128, 256)]
+
+    grids = encode_regions_dense(
+        model=enc, device="cpu", wsi=wsi, coordinates=coords,
+        requested_spacing_um=0.5, target_size=target_size,
+    )
+
+    # Re-read the same regions (deterministic) and encode them directly.
+    from PIL import Image
+
+    geometry = compute_dense_geometry(target_size=target_size, patch_size=enc.patch_size)
+    transform = enc.get_dense_transform()
+    ref_wsi = _FakeWSI(target_h=target_size, target_w=target_size)
+    with torch.inference_mode():
+        for i, loc in enumerate(coords):
+            region = ref_wsi.read_region_at_spacing(
+                loc, 0.5, (target_size, target_size), tolerance=0.05, interpolation="area"
+            )
+            tensor = torch.as_tensor(transform(Image.fromarray(region))).as_subclass(torch.Tensor)
+            padded = pad_image_to_encoded(tensor, geometry, pad_mode="reflect", image_pad_value=None)
+            ref = enc.encode_tiles_dense(padded.unsqueeze(0)).detach().float().cpu().numpy()[0]
+            np.testing.assert_allclose(grids[i], ref, rtol=0, atol=1e-6)
+
+
+def test_encode_regions_dense_empty_coordinates():
+    enc = _encoder()
+    wsi = _FakeWSI(target_h=64, target_w=64)
+    grids = encode_regions_dense(
+        model=enc, device="cpu", wsi=wsi, coordinates=[],
+        requested_spacing_um=0.5, target_size=64,
+    )
+    assert grids.shape == (0, 0, 4, 4)
+    assert wsi.calls == []

{slide2vec-4.6.3 → slide2vec-4.7.0}/tests/test_regression_inference.py

@@ -1578,6 +1578,68 @@ def test_run_pipeline_local_persists_completed_embeddings_before_later_slide_fai
     assert recorded.loc["slide-b", "aggregation_status"] == "tbp"
 
 
+def _drive_persist_callback(monkeypatch, tmp_path, *, model_level, num_samples):
+    """Build an incremental persist callback and feed it `num_samples` completions.
+
+    Returns the list of per-write batch sizes recorded by a stubbed
+    ``update_process_list_after_embedding`` (i.e. how the process_list rewrites
+    were grouped).
+    """
+    process_list_path = tmp_path / "process_list.csv"
+    process_list_path.write_text("sample_id\n", encoding="utf-8")
+
+    monkeypatch.setattr(persist_callbacks, "should_persist_tile_embeddings", lambda *a, **k: True)
+    monkeypatch.setattr(persist_callbacks, "is_hierarchical_preprocessing", lambda *a, **k: False)
+    monkeypatch.setattr(persist_callbacks, "resolved_process_list_output_variant", lambda *a, **k: None)
+    monkeypatch.setattr(persist_callbacks, "TILE_EMBEDDING_FLUSH_INTERVAL", 3)
+
+    def fake_persist_embedded_slide(model, embedded_slide, tiling_result, *, preprocessing, execution):
+        artifact = persist_callbacks.TileEmbeddingArtifact(
+            sample_id=embedded_slide.sample_id,
+            path=tmp_path / f"{embedded_slide.sample_id}.pt",
+            metadata_path=tmp_path / f"{embedded_slide.sample_id}.meta.json",
+            format="pt",
+            feature_dim=4,
+            num_tiles=2,
+        )
+        return artifact, None
+
+    monkeypatch.setattr(persist_callbacks, "persist_embedded_slide", fake_persist_embedded_slide)
+
+    batch_sizes: list[int] = []
+    monkeypatch.setattr(
+        persist_callbacks,
+        "update_process_list_after_embedding",
+        lambda *a, **k: batch_sizes.append(len(k["successful_slides"])),
+    )
+
+    model = SimpleNamespace(name="enc", level=model_level)
+    callback, _, _ = persist_callbacks.build_incremental_persist_callback(
+        model=model,
+        preprocessing=SimpleNamespace(),
+        execution=SimpleNamespace(output_dir=tmp_path),
+        process_list_path=process_list_path,
+    )
+    for i in range(num_samples):
+        callback(make_slide(f"s-{i}"), SimpleNamespace(), SimpleNamespace(sample_id=f"s-{i}"))
+    return batch_sizes
+
+
+def test_incremental_persist_callback_batches_tile_level_process_list_writes(monkeypatch, tmp_path: Path):
+    # Tile-level (many cheap samples): writes are batched at the flush interval,
+    # and the trailing partial batch is left for the caller's final reconciliation
+    # (so 7 completions at interval 3 -> two writes of 3, one sample still buffered).
+    batch_sizes = _drive_persist_callback(monkeypatch, tmp_path, model_level="tile", num_samples=7)
+    assert batch_sizes == [3, 3]
+
+
+def test_incremental_persist_callback_checkpoints_slide_level_every_sample(monkeypatch, tmp_path: Path):
+    # Slide-level (few expensive samples): every completion is checkpointed
+    # immediately so a crash never loses an expensive slide embedding.
+    batch_sizes = _drive_persist_callback(monkeypatch, tmp_path, model_level="slide", num_samples=4)
+    assert batch_sizes == [1, 1, 1, 1]
+
+
 def test_tile_slides_forwards_spacing_at_level_0_to_hs2p(monkeypatch, tmp_path: Path):
     import slide2vec.inference as inference
 
@@ -2532,6 +2594,71 @@ def test_direct_embed_slides_persists_completed_embeddings_before_later_slide_fa
     assert recorded.loc["slide-b", "feature_status"] == "tbp"
     assert recorded.loc["slide-b", "aggregation_status"] == "tbp"
 
+
+def test_direct_embed_slides_single_gpu_reconciles_batched_tile_status_on_clean_run(monkeypatch, tmp_path: Path):
+    # Tile-level single-GPU run with fewer slides than TILE_EMBEDDING_FLUSH_INTERVAL:
+    # the incremental callback buffers every completion and never flushes mid-run,
+    # so feature_status is written only by the end-of-run reconciliation. Without
+    # it, a clean run would leave these rows as "tbp" despite the .pt files existing.
+    pytest.importorskip("torch")
+    import slide2vec.inference as inference
+
+    slides = [make_slide("slide-a"), make_slide("slide-b")]
+    tiling_results = [
+        SimpleNamespace(
+            x=np.array([0, 1]),
+            y=np.array([2, 3]),
+            tile_size_lv0=224,
+            coordinates_npz_path=Path("/tmp/slide-a.coordinates.npz"),
+            coordinates_meta_path=Path("/tmp/slide-a.coordinates.meta.json"),
+        ),
+        SimpleNamespace(
+            x=np.array([4, 5]),
+            y=np.array([6, 7]),
+            tile_size_lv0=224,
+            coordinates_npz_path=Path("/tmp/slide-b.coordinates.npz"),
+            coordinates_meta_path=Path("/tmp/slide-b.coordinates.meta.json"),
+        ),
+    ]
+    process_list_path = tmp_path / "process_list.csv"
+    process_list_path.write_text(
+        "sample_id,annotation,image_path,mask_path,spacing_at_level_0,tiling_status,num_tiles,coordinates_npz_path,coordinates_meta_path,error,traceback\n"
+        "slide-a,tissue,/tmp/slide-a.svs,,,"
+        "success,2,/tmp/slide-a.coordinates.npz,/tmp/slide-a.coordinates.meta.json,,\n"
+        "slide-b,tissue,/tmp/slide-b.svs,,,"
+        "success,2,/tmp/slide-b.coordinates.npz,/tmp/slide-b.coordinates.meta.json,,\n",
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(tiling_pipeline, "prepare_tiled_slides",
+        lambda slide_records, preprocessing, output_dir, num_workers: (slides, tiling_results, process_list_path),
+    )
+    # Clean run: every slide embeds successfully.
+    monkeypatch.setattr(embedding_pipeline, "compute_tile_embeddings_for_slide",
+        lambda *args, **kwargs: np.zeros((2, 4), dtype=np.float32),
+    )
+
+    model = SimpleNamespace(
+        name="uni2",
+        level="tile",
+        _requested_device="cpu",
+        _load_backend=lambda: SimpleNamespace(),
+    )
+
+    inference.embed_slides(
+        model,
+        slides,
+        preprocessing=DEFAULT_PREPROCESSING,
+        execution=ExecutionOptions(output_dir=tmp_path, save_tile_embeddings=True, num_gpus=1),
+    )
+
+    assert (tmp_path / "tile_embeddings" / "slide-a.pt").is_file()
+    assert (tmp_path / "tile_embeddings" / "slide-b.pt").is_file()
+    recorded = pd.read_csv(process_list_path).set_index("sample_id")
+    assert recorded.loc["slide-a", "feature_status"] == "success"
+    assert recorded.loc["slide-b", "feature_status"] == "success"
+
+
 def test_slide_level_pipeline_skips_tile_artifacts_when_save_tile_embeddings_is_false(monkeypatch, tmp_path: Path):
     import slide2vec.inference as inference