slide2vec 6.3.4__tar.gz → 7.0.0__tar.gz

{slide2vec-6.3.4 → slide2vec-7.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: slide2vec
-Version: 6.3.4
+Version: 7.0.0
 Summary: Embedding of whole slide images with Foundation Models
 Author-email: Clément Grisi <clement.grisi@radboudumc.nl>
 License-Expression: Apache-2.0
@@ -15,7 +15,7 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=5.1.0
+Requires-Dist: hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=6.0.0
 Requires-Dist: omegaconf
 Requires-Dist: matplotlib
 Requires-Dist: numpy<2
@@ -75,7 +75,7 @@ Requires-Dist: numpy<2; extra == "fm"
 Requires-Dist: pandas; extra == "fm"
 Requires-Dist: pillow; extra == "fm"
 Requires-Dist: rich; extra == "fm"
-Requires-Dist: hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=5.1.0; extra == "fm"
+Requires-Dist: hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=6.0.0; extra == "fm"
 Requires-Dist: wandb; extra == "fm"
 Requires-Dist: torch<2.8,>=2.3; extra == "fm"
 Requires-Dist: torchvision>=0.18.0; extra == "fm"

{slide2vec-6.3.4 → slide2vec-7.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "slide2vec"
-version = "6.3.4"
+version = "7.0.0"
 description = "Embedding of whole slide images with Foundation Models"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -21,7 +21,7 @@ classifiers = [
     "Programming Language :: Python :: 3.13",
 ]
 dependencies = [
-    "hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=5.1.0",
+    "hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=6.0.0",
     "omegaconf",
     "matplotlib",
     "numpy<2",
@@ -100,7 +100,7 @@ fm = [
     "pandas",
     "pillow",
     "rich",
-    "hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=5.1.0",
+    "hs2p[asap,cucim,openslide,sam2,tifffile,vips]>=6.0.0",
     "wandb",
     "torch>=2.3,<2.8",
     "torchvision>=0.18.0",
@@ -150,7 +150,7 @@ include = ["slide2vec*"]
 
 [tool.setuptools.package-data]
 slide2vec = ["py.typed", "cufile.json"]
-"slide2vec.configs" = ["*.yaml", "models/*.yaml", "preprocessing/*.yaml"]
+"slide2vec.configs" = ["*.yaml"]
 
 [tool.pytest.ini_options]
 addopts = "--cov=slide2vec -m 'not gpu_integration'"
@@ -180,7 +180,7 @@ no_implicit_reexport = true
 max-line-length = 160
 
 [tool.bumpver]
-current_version = "6.3.4"
+current_version = "7.0.0"
 version_pattern = "MAJOR.MINOR.PATCH"
 commit = false
 tag = false

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/__init__.py

@@ -25,13 +25,14 @@ from slide2vec.artifacts import (
     TileEmbeddingArtifact,
 )
 from slide2vec.runtime.dense_encode import DenseEncodeGeometry, DenseEncodeKit
+from slide2vec.runtime.feature_identity import MISSING_FIELD
 from slide2vec.encoders import (
     EncoderProviderDiagnostic,
     list_encoder_provider_diagnostics,
 )
 
 
-__version__ = "6.3.4"
+__version__ = "7.0.0"
 
 __all__ = [
     "Model",
@@ -56,5 +57,6 @@ __all__ = [
     "DenseRegionArtifact",
     "DenseImageArtifact",
     "ImageEmbeddingArtifact",
+    "MISSING_FIELD",
     "__version__",
 ]

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/api.py

@@ -66,19 +66,6 @@ SlideSequence = Sequence[SlideInput]
 TilingResultsInput = Sequence[Any] | Mapping[str, Any]
 
 
-#: Default annotation-mask vocabulary — plain binary tissue tiling. Mirrors hs2p's
-#: shipped default ``{background: 0, tissue: 1}``; leaving it untouched keeps a run
-#: behaving exactly as a tissue-only run. ``min_coverage.tissue`` is the single source
-#: of truth for the tissue threshold (the standalone ``tissue_threshold`` knob is gone).
-#: A :class:`PreprocessingConfig` ``masks`` value is deep-merged over this default, so
-#: callers only state what they override (e.g. ``{"min_coverage": {"tissue": 0.1}}``).
-DEFAULT_MASKS: dict[str, Any] = {
-    "output_mode": "per_annotation",
-    "pixel_mapping": {"background": 0, "tissue": 1},
-    "colors": {"background": None, "tissue": [157, 219, 129]},
-    "min_coverage": {"background": None, "tissue": 0.01},
-}
-
 _REQUESTED_TILE_SIZE_INTERPOLATION = "${tiling.params.requested_tile_size_px}"
 
 
@@ -92,6 +79,7 @@ def _load_default_preprocessing() -> dict[str, dict[str, Any]]:
         ("segmentation", "seg_params"),
         ("filtering", "filter_params"),
         ("preview", "preview"),
+        ("masks", "masks"),
     ):
         section = OmegaConf.to_container(getattr(tiling, config_name), resolve=False)
         if not isinstance(section, dict):
@@ -107,6 +95,13 @@ def _load_default_preprocessing() -> dict[str, dict[str, Any]]:
 #: ``configs/default.yaml`` so Python and YAML entry points share one source.
 DEFAULT_PREPROCESSING = _load_default_preprocessing()
 
+#: Default annotation-mask vocabulary, loaded from ``configs/default.yaml``: plain
+#: binary tissue tiling (``{background: 0, tissue: 1}``) with
+#: ``min_coverage.tissue = 0.1``. ``min_coverage.tissue`` is the single source of truth
+#: for the tissue threshold. A :class:`PreprocessingConfig` ``masks`` value is
+#: deep-merged over this default, so callers only state what they override.
+DEFAULT_MASKS: dict[str, Any] = DEFAULT_PREPROCESSING["masks"]
+
 
 def _deep_merge_dicts(base: Mapping[str, Any], override: Mapping[str, Any]) -> dict[str, Any]:
     """Deep-merge *override* onto a copy of *base* (nested dicts merge key-by-key)."""
@@ -213,7 +208,7 @@ class PreprocessingConfig:
     #: Annotation-mask vocabulary forwarded to hs2p's sampling resolver. Keys:
     #: ``output_mode``, ``pixel_mapping``, ``colors``, ``min_coverage``. A partial
     #: mapping is deep-merged over :data:`DEFAULT_MASKS`, so callers only state what
-    #: they override (e.g. ``{"min_coverage": {"tissue": 0.1}}``). The default
+    #: they override (e.g. ``{"min_coverage": {"tissue": 0.5}}``). The default
     #: ``{background, tissue}`` block is plain tissue tiling; ``min_coverage.tissue``
     #: is the single source of truth for the tissue threshold.
     masks: dict[str, Any] = field(default_factory=dict)
@@ -311,6 +306,9 @@ class PreprocessingConfig:
         return replace(self, mask_backend=mask_backend)
 
 
+#: Accepted values of :attr:`ExecutionOptions.on_image_mismatch`.
+IMAGE_MISMATCH_POLICIES = ("raise", "reencode")
+
 
 @dataclass(frozen=True, kw_only=True)
 class ExecutionOptions:
@@ -347,6 +345,10 @@ class ExecutionOptions:
     #: Return and persist the slide encoder's latent representations (PRISM). Encoders
     #: without latents ignore it.
     save_latents: bool = False
+    #: :meth:`Model.embed_images` only: what to do when an existing image embedding
+    #: records a different source ``image_path`` for its ``sample_id``. ``"raise"``
+    #: (default) refuses to resume; ``"reencode"`` replaces that image's artifacts.
+    on_image_mismatch: str = "raise"
 
     @classmethod
     def from_config(cls, cfg: Any, *, run_on_cpu: bool = False) -> "ExecutionOptions":
@@ -380,8 +382,18 @@ class ExecutionOptions:
         object.__setattr__(self, "output_dtype", normalize_output_dtype(self.output_dtype))
         if resolved_num_gpus < 1:
             raise ValueError("ExecutionOptions.num_gpus must be at least 1")
+        if self.batch_size < 1:
+            raise ValueError("ExecutionOptions.batch_size must be at least 1")
         if self.prefetch_factor < 1:
             raise ValueError("ExecutionOptions.prefetch_factor must be at least 1")
+        if self.num_workers_per_gpu is not None and self.num_workers_per_gpu < 0:
+            raise ValueError("ExecutionOptions.num_workers_per_gpu must be non-negative")
+        if self.on_image_mismatch not in IMAGE_MISMATCH_POLICIES:
+            raise ValueError(
+                "ExecutionOptions.on_image_mismatch must be one of "
+                f"{', '.join(repr(policy) for policy in IMAGE_MISMATCH_POLICIES)}; "
+                f"got {self.on_image_mismatch!r}"
+            )
         cap = cpu_worker_limit()
         cpu_count = os.cpu_count() or 1
         slurm_limit = slurm_cpu_limit()
@@ -699,22 +711,26 @@ class Model:
         *,
         preprocessing: PreprocessingConfig | None,
         execution: ExecutionOptions | None = None,
+        encoded_pixels: bool = True,
     ) -> dict[str, tuple[Any, Any]]:
         """Compare a pooled sidecar's ``compatibility`` block with the current recipe.
 
-        Returns ``{field: (recorded, current)}`` for differing fields held by both
-        identities. Missing fields retain the extraction resume semantics: they
-        cannot be verified and are accepted. The caller owns the policy for an
-        unrecorded identity or a difference.
+        Returns ``{field: (recorded, current)}`` for every required field that the
+        record holds with a different value or does not hold. A field the record does
+        not hold is reported as :data:`slide2vec.MISSING_FIELD`, which is distinct
+        from a recorded ``None``. An empty result means the record proves the current
+        identity. The caller owns the policy for a difference.
 
         Explicitly pass ``preprocessing=None`` for pre-cropped images, whose shipped
         transform determines geometry. Pass a :class:`PreprocessingConfig` for
         tiles read from slides; its unset fields resolve as in pooled extraction.
         ``execution`` defaults to the model's recommended precision and needs no
-        output directory.
+        output directory. The ``transform`` is required unless ``encoded_pixels`` is
+        false: pass ``encoded_pixels=False`` for a zero-tile slide, which encodes no
+        pixels and records no transform.
 
-        Only a recorded ``transform`` loads weights, on an isolated CPU model.
-        Slide and patient models load their registered tile encoder and tile output
+        Comparing the ``transform`` loads weights, on an isolated CPU model. Slide
+        and patient models load their registered tile encoder and tile output
         variant, without loading aggregation weights. This method does not change
         this model's input contract or loaded backend, encode pixels, or write files.
         """
@@ -729,6 +745,7 @@ class Model:
             dict(recorded),
             execution=resolved_execution,
             preprocessing=resolved_preprocessing,
+            encoded_pixels=encoded_pixels,
         )
 
     def prepare_dense_encoder(
@@ -1024,8 +1041,11 @@ class Model:
         decoded, preprocessed with the encoder's **shipped** transform, encoded, and written
         to ``image_embeddings/<sample_id>.pt`` plus a provenance sidecar. The run splits its
         images across all visible GPUs (``execution.num_gpus``); ``num_gpus=1`` encodes
-        fully in-process. Resume is automatic — images whose sidecar already exists are
-        skipped, and a sidecar that records a different feature identity raises. Returns
+        fully in-process. Resume is automatic: an image is skipped only when its sidecar
+        records this ``output_format``, the requested ``image_path`` and the full feature
+        identity. A different recorded feature identity raises; a different recorded
+        ``image_path`` raises unless ``execution.on_image_mismatch="reencode"``; missing
+        provenance re-encodes. Returns
         one :class:`~slide2vec.artifacts.ImageEmbeddingArtifact` per input image, in input
         order.
 

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/artifacts.py

@@ -137,7 +137,8 @@ class ImageEmbeddingArtifact:
         return load_metadata(self.metadata_path)
 
 
-def _validate_output_format(output_format: str) -> str:
+def normalize_output_format(output_format: str) -> str:
+    """The canonical (lowercase) spelling of a supported ``output_format``."""
     normalized = output_format.lower()
     if normalized not in {"pt", "npz"}:
         raise ValueError(f"Unsupported output format: {output_format}")
@@ -403,27 +404,46 @@ def write_dense_image(
     )
 
 
-def image_embedding_paths(
-    output_dir: str | Path, *, sample_id: str, output_format: str
-) -> tuple[Path, Path]:
-    """``(payload_path, sidecar_path)`` for one given-geometry image.
+#: Payload formats an image embedding can be written in; they share one sidecar.
+IMAGE_EMBEDDING_FORMATS = ("pt", "npz")
 
-    ``image_embeddings/<sample_id>.{pt,npz}`` plus ``image_embeddings/<sample_id>.meta.json``.
-    A given-geometry run has no slide, no coordinate and no annotation class — the caller's
-    ``sample_id`` is the whole identity — so the layout is flat and the resume check needs
-    nothing but the sample id.
+
+def image_embeddings_dir(output_dir: str | Path) -> Path:
+    """The resolved ``image_embeddings/`` directory, refusing one that escapes ``output_dir``.
+
+    A run resolves it once and names each image with :func:`image_embedding_names`.
     """
     output_root = Path(output_dir).expanduser().resolve()
-    sample_component = _validate_path_component(sample_id, field="sample_id")
     embeddings_dir = (output_root / "image_embeddings").resolve()
     if not embeddings_dir.is_relative_to(output_root):
         raise ValueError("Image artifact path must stay within output_dir")
+    return embeddings_dir
+
+
+def image_embedding_names(sample_id: str, *, output_format: str) -> tuple[str, str]:
+    """``(payload_name, sidecar_name)`` of one image inside :func:`image_embeddings_dir`."""
+    sample_component = _validate_path_component(sample_id, field="sample_id")
     return (
-        embeddings_dir / f"{sample_component}.{_validate_output_format(output_format)}",
-        embeddings_dir / f"{sample_component}.meta.json",
+        f"{sample_component}.{normalize_output_format(output_format)}",
+        f"{sample_component}.meta.json",
     )
 
 
+def image_embedding_paths(
+    output_dir: str | Path, *, sample_id: str, output_format: str
+) -> tuple[Path, Path]:
+    """``(payload_path, sidecar_path)`` for one given-geometry image.
+
+    ``image_embeddings/<sample_id>.{pt,npz}`` plus ``image_embeddings/<sample_id>.meta.json``.
+    A given-geometry run has no slide, no coordinate and no annotation class — the caller's
+    ``sample_id`` is the whole identity — so the layout is flat. Both payload formats share
+    the sidecar, which records the ``format`` it certifies.
+    """
+    payload_name, sidecar_name = image_embedding_names(sample_id, output_format=output_format)
+    embeddings_dir = image_embeddings_dir(output_dir)
+    return embeddings_dir / payload_name, embeddings_dir / sidecar_name
+
+
 def write_image_embedding(
     embedding,
     *,
@@ -440,7 +460,7 @@ def write_image_embedding(
     ROI write follows (:func:`write_dense_region`), and the reason a rank can be killed
     mid-shard without poisoning the output.
     """
-    output_format = _validate_output_format(output_format)
+    output_format = normalize_output_format(output_format)
     payload_path, metadata_path = image_embedding_paths(
         output_dir, sample_id=sample_id, output_format=output_format
     )
@@ -520,7 +540,7 @@ def write_tile_embeddings(
     tile_index: Any | None = None,
     annotation: str | None = None,
 ) -> TileEmbeddingArtifact:
-    output_format = _validate_output_format(output_format)
+    output_format = normalize_output_format(output_format)
     artifact_path, metadata_path = _setup_artifact_paths(
         output_dir, tile_embeddings_subdir(annotation), sample_id, output_format
     )
@@ -563,7 +583,7 @@ def write_tile_embedding_metadata(
     metadata: dict[str, Any] | None = None,
     annotation: str | None = None,
 ) -> Path:
-    output_format = _validate_output_format(output_format)
+    output_format = normalize_output_format(output_format)
     _, metadata_path = _setup_artifact_paths(
         output_dir, tile_embeddings_subdir(annotation), sample_id, output_format
     )
@@ -589,7 +609,7 @@ def write_slide_embeddings(
     latents: Any | None = None,
     annotation: str | None = None,
 ) -> SlideEmbeddingArtifact:
-    output_format = _validate_output_format(output_format)
+    output_format = normalize_output_format(output_format)
     annotation = structural_artifact_annotation(annotation)
     artifact_path, metadata_path = _setup_artifact_paths(
         output_dir, slide_embeddings_subdir(annotation), sample_id, output_format
@@ -638,7 +658,7 @@ def write_patient_embeddings(
     metadata: dict[str, Any] | None = None,
     num_slides: int = 0,
 ) -> PatientEmbeddingArtifact:
-    output_format = _validate_output_format(output_format)
+    output_format = normalize_output_format(output_format)
     artifact_path, metadata_path = _setup_artifact_paths(
         output_dir, "patient_embeddings", patient_id, output_format
     )
@@ -677,7 +697,7 @@ def write_hierarchical_embeddings(
     metadata: dict[str, Any] | None = None,
     annotation: str | None = None,
 ) -> HierarchicalEmbeddingArtifact:
-    output_format = _validate_output_format(output_format)
+    output_format = normalize_output_format(output_format)
     annotation = structural_artifact_annotation(annotation)
     artifact_path, metadata_path = _setup_artifact_paths(
         output_dir, hierarchical_embeddings_subdir(annotation), sample_id, output_format

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/configs/default.yaml

@@ -9,7 +9,7 @@ save_tiles: false # save extracted tiles as {sample_id}.tiles.tar in addition to
 seed: 0 # seed for reproducibility
 
 model:
-  name: # preset model name; see docs/models.md for supported request strings
+  name: # preset model name; see docs/models.rst for supported request strings
   output_variant: # requested output variant for presets that expose multiple outputs
   batch_size: 32
   save_tile_embeddings: false # whether to save tile embeddings alongside the pooled slide embedding when level is "slide"
@@ -18,7 +18,7 @@ model:
   allow_non_recommended_settings: false # when true, non-recommended spacing / tile size / precision combinations warn instead of erroring
 
 tiling:
-  on_the_fly: true # read tiles directly from WSI during embedding (requires cucim backend)
+  on_the_fly: true # read tiles directly from the WSI during embedding; when false, embedding reads tiles from {sample_id}.tiles.tar
   gpu_decode: false # GPU-accelerated batch decoding via device="cuda" in cucim read_region; set true to opt in when the runtime is configured for it (experimental, may not be stable across all environments)
   adaptive_batching: false # when true, vary batch size to align with super tile boundaries (avoids redundant reads but batch size fluctuates)
   use_supertiles: true # group tiles into 8x8/4x4/2x2 super tile reads to reduce WSI read calls (on-the-fly path only)

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/configs/resources.py

@@ -1,6 +1,4 @@
-from contextlib import contextmanager
 from pathlib import Path
-from typing import Iterator
 
 
 def config_resource(*parts: str):
@@ -16,8 +14,3 @@ def load_config(*parts: str):
     resource = config_resource(*parts)
     with resource.open("r", encoding="utf-8") as handle:
         return OmegaConf.load(handle)
-
-
-@contextmanager
-def config_path(*parts: str) -> Iterator[Path]:
-    yield config_resource(*parts)

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/data/tile_reader.py

@@ -429,7 +429,6 @@ class OnTheFlyHierarchicalBatchCollator:
     ):
         self._region_index = np.asarray(region_index, dtype=np.int32)
         self._subtile_index_within_region = np.asarray(subtile_index_within_region, dtype=np.int32)
-        self._tiles_per_region = int(self._subtile_index_within_region.max()) + 1 if len(self._subtile_index_within_region) else 0
         self._read_tile_size_px = int(read_tile_size_px)
         self._requested_tile_size_px = int(requested_tile_size_px)
         self._reader = WSIRegionReader(

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/distributed/direct_embed_worker.py

@@ -17,8 +17,12 @@ def main(argv=None) -> int:
     import torch
 
     import slide2vec.distributed as distributed
-    import slide2vec.inference as inference
     from slide2vec.api import Model
+    from slide2vec.runtime.embedding_pipeline import (
+        compute_embedded_slides,
+        compute_hierarchical_embedding_shard_for_slide,
+        compute_tile_embeddings_for_slide,
+    )
     from slide2vec.runtime.hierarchical import (
         build_hierarchical_index,
         is_hierarchical_preprocessing,
@@ -56,11 +60,9 @@ def main(argv=None) -> int:
     )
     from slide2vec.runtime.embedding import tiling_result_annotation
     from slide2vec.runtime.feature_identity import transform_record
+    from slide2vec.runtime.manifest import load_successful_tiled_slides
 
-    load_successful_tiled_slides_fn = getattr(inference, "load_successful_tiled_slides", None)
-    if not callable(load_successful_tiled_slides_fn):
-        from slide2vec.runtime.manifest import load_successful_tiled_slides as load_successful_tiled_slides_fn
-    slide_records, tiling_results = load_successful_tiled_slides_fn(output_dir)
+    slide_records, tiling_results = load_successful_tiled_slides(output_dir)
     # Key by the composite (sample_id, annotation) work unit so a multi-class slide's sibling
     # classes never overwrite each other; flat units collapse to the bare sample_id key.
     paired_by_unit = {
@@ -95,16 +97,7 @@ def main(argv=None) -> int:
                     tile_size_lv0=int(geometry["tile_size_lv0"]),
                 )
                 flat_indices = np.array_split(index.flat_index, world_size)[global_rank]
-                compute_hierarchical_embedding_shard_for_slide_fn = getattr(
-                    inference,
-                    "_compute_hierarchical_embedding_shard_for_slide",
-                    None,
-                )
-                if not callable(compute_hierarchical_embedding_shard_for_slide_fn):
-                    from slide2vec.runtime.embedding_pipeline import (
-                        compute_hierarchical_embedding_shard_for_slide as compute_hierarchical_embedding_shard_for_slide_fn,
-                    )
-                shard_indices, tile_embeddings = compute_hierarchical_embedding_shard_for_slide_fn(
+                shard_indices, tile_embeddings = compute_hierarchical_embedding_shard_for_slide(
                     loaded,
                     slide,
                     tiling_result,
@@ -122,16 +115,7 @@ def main(argv=None) -> int:
             else:
                 num_tiles = len(tiling_result.x)
                 tile_indices = np.array_split(np.arange(num_tiles, dtype=np.int64), world_size)[global_rank]
-                compute_tile_embeddings_for_slide_fn = getattr(
-                    inference,
-                    "_compute_tile_embeddings_for_slide",
-                    None,
-                )
-                if not callable(compute_tile_embeddings_for_slide_fn):
-                    from slide2vec.runtime.embedding_pipeline import (
-                        compute_tile_embeddings_for_slide as compute_tile_embeddings_for_slide_fn,
-                    )
-                tile_embeddings = compute_tile_embeddings_for_slide_fn(
+                tile_embeddings = compute_tile_embeddings_for_slide(
                     loaded,
                     model,
                     slide,
@@ -170,10 +154,7 @@ def main(argv=None) -> int:
             )
             torch.save(payload, coordination_dir / f"{stem}.embedded.pt")
 
-        compute_embedded_slides_fn = getattr(inference, "_compute_embedded_slides", None)
-        if not callable(compute_embedded_slides_fn):
-            from slide2vec.runtime.embedding_pipeline import compute_embedded_slides as compute_embedded_slides_fn
-        compute_embedded_slides_fn(
+        compute_embedded_slides(
             model,
             assigned_slides,
             assigned_tiling_results,

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/distributed/image_worker.py

@@ -55,7 +55,7 @@ def main(argv=None) -> int:
         def _on_batch(count: int) -> None:
             emit_progress("images.batch.finished", rank=rank.global_rank, images=int(count))
 
-        run_image_shard(
+        artifacts = run_image_shard(
             shard,
             loaded=loaded,
             out_dir=output_dir,
@@ -67,7 +67,17 @@ def main(argv=None) -> int:
             num_workers=execution.resolved_image_num_workers_per_gpu(),
             prefetch_factor=int(execution.prefetch_factor),
             on_batch=_on_batch,
+            # The parent planned what to invalidate; the shard loop deletes each image's
+            # files right before writing it, so a failed launch, load or sibling rank costs
+            # at most the images in flight.
+            stale=request["stale"],
         )
+    # The parent collects artifacts without reading sidecars: report the vector width.
+    result_path = Path(request["result_dir"]) / f"image_result.rank{rank.global_rank}.json"
+    result_path.write_text(
+        json.dumps({"feature_dim": int(artifacts[0].feature_dim), "images": len(artifacts)}),
+        encoding="utf-8",
+    )
     return 0
 
 

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/distributed/pipeline_worker.py

@@ -16,8 +16,9 @@ def get_args_parser(add_help: bool = True) -> argparse.ArgumentParser:
 
 def main(argv=None) -> int:
     import slide2vec.distributed as distributed
-    import slide2vec.inference as inference
     from slide2vec.api import Model
+    from slide2vec.runtime.embedding_pipeline import compute_embedded_slides
+    from slide2vec.runtime.manifest import load_successful_tiled_slides
     from slide2vec.runtime.persist_callbacks import build_incremental_persist_callback
     from slide2vec.progress import JsonlProgressReporter, activate_progress_reporter
     from slide2vec.runtime.serialization import deserialize_execution, deserialize_preprocessing
@@ -44,10 +45,7 @@ def main(argv=None) -> int:
     execution = deserialize_execution(request["execution"])
     model._declare_encoder_input(preprocessing, emit_run_info=False)
     tiling_input_dir = Path(request.get("tiling_input_dir", str(output_dir)))
-    load_successful_tiled_slides_fn = getattr(inference, "load_successful_tiled_slides", None)
-    if not callable(load_successful_tiled_slides_fn):
-        from slide2vec.runtime.manifest import load_successful_tiled_slides as load_successful_tiled_slides_fn
-    slide_records, tiling_results = load_successful_tiled_slides_fn(tiling_input_dir)
+    slide_records, tiling_results = load_successful_tiled_slides(tiling_input_dir)
     # Each (sample_id, annotation) row is an independent work unit; key by the composite so a
     # multi-class slide's sibling classes never overwrite each other. Flat units (None / tissue /
     # merged) encode to the bare sample_id, byte-identical to pre-#168 single-class runs.
@@ -83,17 +81,13 @@ def main(argv=None) -> int:
     )
     context = activate_progress_reporter(reporter) if reporter is not None else nullcontext()
     with context:
-        build_incremental_persist_callback_fn = getattr(inference, "_build_incremental_persist_callback", build_incremental_persist_callback)
-        persist_callback, _, _ = build_incremental_persist_callback_fn(
+        persist_callback, _, _ = build_incremental_persist_callback(
             model=model,
             preprocessing=preprocessing,
             execution=execution,
             process_list_path=None,
         )
-        compute_embedded_slides_fn = getattr(inference, "_compute_embedded_slides", None)
-        if not callable(compute_embedded_slides_fn):
-            from slide2vec.runtime.embedding_pipeline import compute_embedded_slides as compute_embedded_slides_fn
-        compute_embedded_slides_fn(
+        compute_embedded_slides(
             model,
             assigned_slides,
             assigned_tiling_results,

{slide2vec-6.3.4 → slide2vec-7.0.0}/slide2vec/runtime/artifacts_collect.py

@@ -7,7 +7,7 @@ import pandas as pd
 from hs2p import SlideSpec
 from hs2p.fileops import read_csv_keyed_by_sample_id
 
-from slide2vec.api import EmbeddedSlide, ExecutionOptions, PreprocessingConfig
+from slide2vec.api import ExecutionOptions, PreprocessingConfig
 from slide2vec.artifacts import (
     HierarchicalEmbeddingArtifact,
     normalize_artifact_annotation,
@@ -15,7 +15,6 @@ from slide2vec.artifacts import (
     TileEmbeddingArtifact,
 )
 from slide2vec.runtime.distributed_stage import run_distributed_embedding_stage
-from slide2vec.runtime.embedding_persist import persist_embedded_slide
 from slide2vec.runtime.feature_identity import (
     deferred_transform_record,
     pooled_feature_identity,
@@ -34,34 +33,6 @@ from slide2vec.runtime.process_list import resolved_process_list_output_variant
 from slide2vec.progress import emit_progress
 
 
-def collect_local_pipeline_artifacts(
-    *,
-    model,
-    embedded_slides: Sequence[EmbeddedSlide],
-    tiling_results,
-    preprocessing: PreprocessingConfig,
-    execution: ExecutionOptions,
-) -> tuple[list[TileEmbeddingArtifact], list[HierarchicalEmbeddingArtifact], list[SlideEmbeddingArtifact]]:
-    tile_artifacts: list[TileEmbeddingArtifact] = []
-    hierarchical_artifacts: list[HierarchicalEmbeddingArtifact] = []
-    slide_artifacts: list[SlideEmbeddingArtifact] = []
-    for embedded_slide, tiling_result in zip(embedded_slides, tiling_results):
-        tile_artifact, slide_artifact = persist_embedded_slide(
-            model,
-            embedded_slide,
-            tiling_result,
-            preprocessing=preprocessing,
-            execution=execution,
-        )
-        if isinstance(tile_artifact, HierarchicalEmbeddingArtifact):
-            hierarchical_artifacts.append(tile_artifact)
-        elif tile_artifact is not None:
-            tile_artifacts.append(tile_artifact)
-        if slide_artifact is not None:
-            slide_artifacts.append(slide_artifact)
-    return tile_artifacts, hierarchical_artifacts, slide_artifacts
-
-
 def _embeddable_annotation_groups(
     process_list_path: Path,
 ) -> dict[str, list[str | None]]:
@@ -216,6 +187,11 @@ def collect_distributed_pipeline_artifacts(
         )
     slide_by_sample_id = {slide.sample_id: slide for slide in successful_slides}
     live_updated_sample_ids: set[str] = set()
+    # Pending classes not yet reported finished, per sample. Resume can requeue a class whose
+    # payload is still on disk (unverifiable provenance), so file existence alone is not completion.
+    unfinished_annotations: dict[str, set[str | None]] = {}
+    for slide, annotation in zip(pending_slides, pending_annotations):
+        unfinished_annotations.setdefault(slide.sample_id, set()).add(annotation)
 
     def _update_process_list_for_finished_slide(event) -> None:
         if getattr(event, "kind", None) != "embedding.slide.finished":
@@ -225,6 +201,10 @@ def collect_distributed_pipeline_artifacts(
         slide = slide_by_sample_id.get(sample_id)
         if slide is None or sample_id in live_updated_sample_ids:
             return
+        unfinished = unfinished_annotations.get(sample_id, set())
+        unfinished.discard(_normalized_row_annotation(payload.get("annotation")))
+        if unfinished:
+            return
         # A multi-class slide emits one finished event per (sample_id, annotation) item, and the
         # other classes may still be running on another rank. Wait until *every* annotation row for
         # the sample is persisted on disk before the whole-sample rewrite, so we never (a) load a