slide2vec 4.0.1__tar.gz → 4.0.3__tar.gz

{slide2vec-4.0.1 → slide2vec-4.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: slide2vec
-Version: 4.0.1
+Version: 4.0.3
 Summary: Embedding of whole slide images with Foundation Models
 Author-email: Clément Grisi <clement.grisi@radboudumc.nl>
 License-Expression: Apache-2.0
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hs2p[asap,cucim,openslide,vips]>=3.1.3
+Requires-Dist: hs2p[asap,cucim,openslide,vips]>=3.1.4
 Requires-Dist: omegaconf
 Requires-Dist: matplotlib
 Requires-Dist: numpy<2
@@ -63,7 +63,7 @@ Requires-Dist: numpy<2; extra == "fm"
 Requires-Dist: pandas; extra == "fm"
 Requires-Dist: pillow; extra == "fm"
 Requires-Dist: rich; extra == "fm"
-Requires-Dist: hs2p[asap,cucim,openslide,vips]>=3.1.3; extra == "fm"
+Requires-Dist: hs2p[asap,cucim,openslide,vips]>=3.1.4; extra == "fm"
 Requires-Dist: wandb; extra == "fm"
 Requires-Dist: torch<2.8,>=2.3; extra == "fm"
 Requires-Dist: torchvision>=0.18.0; extra == "fm"

{slide2vec-4.0.1 → slide2vec-4.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "slide2vec"
-version = "4.0.1"
+version = "4.0.3"
 description = "Embedding of whole slide images with Foundation Models"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -21,7 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "hs2p[asap,cucim,openslide,vips]>=3.1.3",
+    "hs2p[asap,cucim,openslide,vips]>=3.1.4",
     "omegaconf",
     "matplotlib",
     "numpy<2",
@@ -85,7 +85,7 @@ fm = [
     "pandas",
     "pillow",
     "rich",
-    "hs2p[asap,cucim,openslide,vips]>=3.1.3",
+    "hs2p[asap,cucim,openslide,vips]>=3.1.4",
     "wandb",
     "torch>=2.3,<2.8",
     "torchvision>=0.18.0",
@@ -154,7 +154,7 @@ no_implicit_reexport = true
 max-line-length = 160
 
 [tool.bumpver]
-current_version = "4.0.1"
+current_version = "4.0.3"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit = false       # We do version bumping in CI, not as a commit
 tag = false          # Git tag already exists — we don't auto-tag

{slide2vec-4.0.1 → slide2vec-4.0.3}/slide2vec/__init__.py

@@ -2,7 +2,7 @@ from slide2vec.api import EmbeddedSlide, ExecutionOptions, Model, Pipeline, Prep
 from slide2vec.artifacts import HierarchicalEmbeddingArtifact, SlideEmbeddingArtifact, TileEmbeddingArtifact
 
 
-__version__ = "4.0.1"
+__version__ = "4.0.3"
 
 __all__ = [
     "Model",

{slide2vec-4.0.1 → slide2vec-4.0.3}/slide2vec/api.py

@@ -22,7 +22,7 @@ from slide2vec.encoders.validation import validate_encoder_config
 from slide2vec.model_settings import canonicalize_model_name, normalize_precision_name
 from slide2vec.progress import emit_progress
 from slide2vec.runtime_types import LoadedModel
-from slide2vec.utils.utils import slurm_cpu_limit
+from slide2vec.utils.utils import cpu_worker_limit, slurm_cpu_limit
 
 PathLike = str | Path
 
@@ -120,8 +120,8 @@ class ExecutionOptions:
     output_dir: Path | None = None
     output_format: str = "pt"
     batch_size: int = 1
-    num_workers: int = 0
-    num_preprocessing_workers: int = 8
+    num_workers: int | None = None
+    num_preprocessing_workers: int | None = None
     num_gpus: int | None = None
     precision: str | None = None
     prefetch_factor: int = 4
@@ -140,8 +140,12 @@ class ExecutionOptions:
             output_dir=Path(cfg.output_dir),
             output_format="pt",
             batch_size=int(cfg.model.batch_size),
-            num_workers=int(num_workers),
-            num_preprocessing_workers=int(cfg.speed.num_preprocessing_workers),
+            num_workers=int(num_workers) if num_workers is not None else None,
+            num_preprocessing_workers=(
+                int(cfg.speed.num_preprocessing_workers)
+                if cfg.speed.num_preprocessing_workers is not None
+                else None
+            ),
             num_gpus=1 if run_on_cpu else (int(configured_num_gpus) if configured_num_gpus is not None else None),
             precision="fp32" if run_on_cpu else requested_precision,
             prefetch_factor=prefetch_factor,
@@ -158,24 +162,33 @@ class ExecutionOptions:
             raise ValueError("ExecutionOptions.num_gpus must be at least 1")
         if self.prefetch_factor < 1:
             raise ValueError("ExecutionOptions.prefetch_factor must be at least 1")
+        cap = cpu_worker_limit()
         cpu_count = os.cpu_count() or 1
         slurm_limit = slurm_cpu_limit()
-        cap = min(cpu_count, slurm_limit) if slurm_limit is not None else cpu_count
-        capped_num_workers = min(self.num_workers, cap)
-        capped_num_preprocessing_workers = min(self.num_preprocessing_workers, cap)
-        object.__setattr__(self, "num_workers", capped_num_workers)
+        capped_num_preprocessing_workers = (
+            cap if self.num_preprocessing_workers is None else min(self.num_preprocessing_workers, cap)
+        )
         object.__setattr__(self, "num_preprocessing_workers", capped_num_preprocessing_workers)
         logger = logging.getLogger(__name__)
         cap_source = f"slurm_cpu_limit={slurm_limit}" if slurm_limit is not None else f"cpu_count={cpu_count}"
+        resolved_num_workers = self.resolved_num_workers()
+        num_workers_label = (
+            f"{resolved_num_workers} (requested=auto)"
+            if self.num_workers is None
+            else str(resolved_num_workers)
+        )
         logger.info(
-            "ExecutionOptions: num_workers=%d, num_preprocessing_workers=%d "
-            "(cap=%d via %s)",
-            capped_num_workers,
+            "ExecutionOptions: num_workers=%s, num_preprocessing_workers=%d "
+            "(preprocessing cap=%d via %s)",
+            num_workers_label,
             capped_num_preprocessing_workers,
             cap,
             cap_source,
         )
 
+    def resolved_num_workers(self) -> int:
+        return cpu_worker_limit() if self.num_workers is None else int(self.num_workers)
+
     def with_output_dir(self, output_dir: PathLike | None) -> "ExecutionOptions":
         if output_dir is None:
             return self

{slide2vec-4.0.1 → slide2vec-4.0.3}/slide2vec/configs/default.yaml

@@ -66,9 +66,9 @@ tiling:
 
 speed:
   precision: # model inference precision ["fp32", "fp16", "bf16"]; if not set, determined automatically based on model recommendations
-  num_dataloader_workers: 8 # number of DataLoader worker processes for reading tiles during embedding (tar path); on-the-fly path derives this automatically from cpu_count // speed.num_cucim_workers
+  num_dataloader_workers: # number of DataLoader worker processes for reading tiles during embedding; defaults to auto (job CPU budget, except cuCIM on-the-fly uses cpu_budget // speed.num_cucim_workers)
   num_gpus: # number of GPUs to use for feature extraction; defaults to all available GPUs
-  num_preprocessing_workers: 8 # number of workers for hs2p tiling (WSI reading, JPEG encoding, tar writing)
+  num_preprocessing_workers: # number of workers for hs2p tiling (WSI reading, JPEG encoding, tar writing); defaults to the runtime CPU budget capped at 64
   num_cucim_workers: 4 # number of internal cucim threads per read_region call (embedding path, on-the-fly only); DataLoader workers are auto-set to cpu_count // num_cucim_workers
   prefetch_factor_embedding: 4 # prefetch factor for tile embedding dataloaders
   persistent_workers_embedding: true # keep DataLoader workers alive across epochs/batches

{slide2vec-4.0.1 → slide2vec-4.0.3}/slide2vec/data/tile_reader.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from contextlib import nullcontext
 import time
 from pathlib import Path
 
@@ -6,7 +7,9 @@ import numpy as np
 import torch
 
 from hs2p import TilingResult
+from hs2p.utils.stderr import run_with_filtered_stderr
 from hs2p.wsi.streaming.plans import build_supertile_index
+from slide2vec.utils.log_utils import suppress_c_stderr
 
 
 class SuperTileBatchSampler:
@@ -139,15 +142,17 @@ class WSITileReader:
                 torch.empty((0, 3, ts, ts), dtype=torch.uint8),
                 {"reader_open_ms": 0.0, "reader_read_ms": 0.0},
             )
-        was_closed = self._reader is None
-        open_start = time.perf_counter()
-        self._ensure_open()
-        reader_open_ms = (time.perf_counter() - open_start) * 1000.0 if was_closed else 0.0
-        read_start = time.perf_counter()
-        if self._use_supertiles:
-            tensor = self._read_batch_supertiles(tile_indices)
-        else:
-            tensor = self._read_batch_simple(tile_indices)
+        stderr_context = suppress_c_stderr() if self._backend == "cucim" else nullcontext()
+        with stderr_context:
+            was_closed = self._reader is None
+            open_start = time.perf_counter()
+            self._ensure_open()
+            reader_open_ms = (time.perf_counter() - open_start) * 1000.0 if was_closed else 0.0
+            read_start = time.perf_counter()
+            if self._use_supertiles:
+                tensor = self._read_batch_supertiles(tile_indices)
+            else:
+                tensor = self._read_batch_simple(tile_indices)
         reader_read_ms = (time.perf_counter() - read_start) * 1000.0
         return tensor, {"reader_open_ms": reader_open_ms, "reader_read_ms": reader_read_ms}
 
@@ -260,11 +265,16 @@ class OnTheFlyBatchTileCollator:
                 torch.empty((0, 3, self.tile_size, self.tile_size), dtype=torch.uint8),
                 {"worker_batch_ms": 0.0, "reader_open_ms": 0.0, "reader_read_ms": 0.0},
             )
-        worker_start = time.perf_counter()
-        tile_indices = np.asarray(batch_indices, dtype=np.int64)
-        tensor, timing = self._reader.read_batch_with_timing(tile_indices)
-        timing["worker_batch_ms"] = (time.perf_counter() - worker_start) * 1000.0
-        return torch.as_tensor(tile_indices, dtype=torch.long), tensor, timing
+        def _run_batch():
+            worker_start = time.perf_counter()
+            tile_indices = np.asarray(batch_indices, dtype=np.int64)
+            tensor, timing = self._reader.read_batch_with_timing(tile_indices)
+            timing["worker_batch_ms"] = (time.perf_counter() - worker_start) * 1000.0
+            return torch.as_tensor(tile_indices, dtype=torch.long), tensor, timing
+
+        if getattr(self._reader, "_backend", None) == "cucim":
+            return run_with_filtered_stderr(_run_batch)
+        return _run_batch()
 
 
 class WSIRegionReader:
@@ -320,12 +330,14 @@ class WSIRegionReader:
                 torch.empty((0, 3, self._region_size_px, self._region_size_px), dtype=torch.uint8),
                 {"reader_open_ms": 0.0, "reader_read_ms": 0.0},
             )
-        was_closed = self._reader is None
-        open_start = time.perf_counter()
-        self._ensure_open()
-        reader_open_ms = (time.perf_counter() - open_start) * 1000.0 if was_closed else 0.0
-        read_start = time.perf_counter()
-        regions = self._read_regions_batch(locations)
+        stderr_context = suppress_c_stderr() if self._backend == "cucim" else nullcontext()
+        with stderr_context:
+            was_closed = self._reader is None
+            open_start = time.perf_counter()
+            self._ensure_open()
+            reader_open_ms = (time.perf_counter() - open_start) * 1000.0 if was_closed else 0.0
+            read_start = time.perf_counter()
+            regions = self._read_regions_batch(locations)
         reader_read_ms = (time.perf_counter() - read_start) * 1000.0
         batch = np.stack([np.asarray(region)[:, :, :3] for region in regions], axis=0)
         tensor = torch.from_numpy(batch).permute(0, 3, 1, 2).contiguous()
@@ -385,17 +397,22 @@ class OnTheFlyHierarchicalBatchCollator:
                 torch.empty((0, 3, self._tile_size, self._tile_size), dtype=torch.uint8),
                 {"worker_batch_ms": 0.0, "reader_open_ms": 0.0, "reader_read_ms": 0.0},
             )
-        worker_start = time.perf_counter()
-        flat_indices = np.asarray(batch_indices, dtype=np.int64)
-        requested_regions = self._region_index[flat_indices]
-        unique_regions, inverse = np.unique(requested_regions, return_inverse=True)
-        locations = [self._region_locations[int(region)] for region in unique_regions]
-        region_tensor, timing = self._reader.read_batch_with_timing(locations)
-        unfolded = _unfold_region_tensor_uint8(region_tensor, self._tile_size)
-        subtile_indices = self._subtile_index_within_region[flat_indices]
-        out = unfolded[torch.as_tensor(inverse, dtype=torch.long), torch.as_tensor(subtile_indices, dtype=torch.long)]
-        timing["worker_batch_ms"] = (time.perf_counter() - worker_start) * 1000.0
-        return torch.as_tensor(flat_indices, dtype=torch.long), out, timing
+        def _run_batch():
+            worker_start = time.perf_counter()
+            flat_indices = np.asarray(batch_indices, dtype=np.int64)
+            requested_regions = self._region_index[flat_indices]
+            unique_regions, inverse = np.unique(requested_regions, return_inverse=True)
+            locations = [self._region_locations[int(region)] for region in unique_regions]
+            region_tensor, timing = self._reader.read_batch_with_timing(locations)
+            unfolded = _unfold_region_tensor_uint8(region_tensor, self._tile_size)
+            subtile_indices = self._subtile_index_within_region[flat_indices]
+            out = unfolded[torch.as_tensor(inverse, dtype=torch.long), torch.as_tensor(subtile_indices, dtype=torch.long)]
+            timing["worker_batch_ms"] = (time.perf_counter() - worker_start) * 1000.0
+            return torch.as_tensor(flat_indices, dtype=torch.long), out, timing
+
+        if getattr(self._reader, "_backend", None) == "cucim":
+            return run_with_filtered_stderr(_run_batch)
+        return _run_batch()
 
 
 def _unfold_region_tensor_uint8(region_tensor: torch.Tensor, tile_size: int) -> torch.Tensor: