scib-metrics 0.5.4__tar.gz → 0.5.6__tar.gz

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     hooks:
       - id: prettier
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.6
+    rev: v0.12.2
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/CHANGELOG.md

@@ -10,6 +10,33 @@ and this project adheres to [Semantic Versioning][].
 
 ## 0.6.0 (unreleased)
 
+## 0.5.6 (2025-07-08)
+
+### Added
+
+- Add BRAS to Benchmarker as default, instead of regular silhouette batch {pr}`217`
+- Added the option to manually set the KNN graphs before running a benchmarker.
+
+### Changed
+
+- Changed default of min_max_scale in {func}`scib_metrics.benchmark.get_results` to False {pr}`215`.
+
+### Fixed
+
+- Reverted Skip labels before loop {pr}`180`, which caused wrong selection of clusters {pr}`213`.
+
+## 0.5.5 (2025-06-03)
+
+### Added
+
+- Add batch removal adapted silhouette (BRAS) metric ({func}`scib_metrics.metrics.bras`) {pr}`197`, which addresses limitations of silhouette for scoring batch effect removal.
+- Add cosine distance implementation required for BRAS.
+
+### Changed
+
+- Changed {func}`scib_metrics.utils.cdist` to support cosine distance.
+- Changed silhouette-related functions to be compatible with adaptions required for BRAS.
+
 ## 0.5.4 (2025-04-23)
 
 ### Fixed

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scib-metrics
-Version: 0.5.4
+Version: 0.5.6
 Summary: Accelerated and Python-only scIB metrics
 Project-URL: Documentation, https://scib-metrics.readthedocs.io/
 Project-URL: Source, https://github.com/yoseflab/scib-metrics

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/docs/api.md

@@ -42,6 +42,7 @@ scib_metrics.ilisi_knn(...)
     pcr_comparison
     silhouette_label
     silhouette_batch
+    bras
     ilisi_knn
     clisi_knn
     kbet

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/docs/references.bib

@@ -36,3 +36,13 @@
   pages = {43--49},
   publisher = {Springer Science and Business Media {LLC}}
 }
+
+@article{rautenstrauch2025,
+  title = {Metrics Matter: Why We Need to Stop Using Silhouette in Single-Cell Benchmarking},
+  author = {Pia Rautenstrauch and Uwe Ohler},
+  doi = {10.1101/2025.01.21.634098},
+  year = {2025},
+  month = jan,
+  journal = {bioRxiv},
+  publisher = {Cold Spring Harbor Laboratory}
+}

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/pyproject.toml

@@ -5,7 +5,7 @@ requires = ["hatchling"]
 
 [project]
 name = "scib-metrics"
-version = "0.5.4"
+version = "0.5.6"
 description = "Accelerated and Python-only scIB metrics"
 readme = "README.md"
 requires-python = ">=3.10"

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/src/scib_metrics/__init__.py

@@ -15,6 +15,7 @@ from .metrics import (
     pcr_comparison,
     silhouette_batch,
     silhouette_label,
+    bras,
 )
 from ._settings import settings
 
@@ -25,6 +26,7 @@ __all__ = [
     "pcr_comparison",
     "silhouette_label",
     "silhouette_batch",
+    "bras",
     "ilisi_knn",
     "clisi_knn",
     "lisi_knn",

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/src/scib_metrics/benchmark/_core.py

@@ -42,6 +42,7 @@ metric_name_cleaner = {
     "clisi_knn": "cLISI",
     "ilisi_knn": "iLISI",
     "kbet_per_label": "KBET",
+    "bras": "BRAS",
     "graph_connectivity": "Graph connectivity",
     "pcr_comparison": "PCR comparison",
 }
@@ -72,7 +73,7 @@ class BatchCorrection:
     parameters, such as `X` or `labels`.
     """
 
-    silhouette_batch: MetricType = True
+    bras: MetricType = True
     ilisi_knn: MetricType = True
     kbet_per_label: MetricType = True
     graph_connectivity: MetricType = True
@@ -88,7 +89,7 @@ class MetricAnnDataAPI(Enum):
     silhouette_label = lambda ad, fn: fn(ad.X, ad.obs[_LABELS])
     clisi_knn = lambda ad, fn: fn(ad.uns["90_neighbor_res"], ad.obs[_LABELS])
     graph_connectivity = lambda ad, fn: fn(ad.uns["15_neighbor_res"], ad.obs[_LABELS])
-    silhouette_batch = lambda ad, fn: fn(ad.X, ad.obs[_LABELS], ad.obs[_BATCH])
+    bras = lambda ad, fn: fn(ad.X, ad.obs[_LABELS], ad.obs[_BATCH])
     pcr_comparison = lambda ad, fn: fn(ad.obsm[_X_PRE], ad.X, ad.obs[_BATCH], categorical=True)
     ilisi_knn = lambda ad, fn: fn(ad.uns["90_neighbor_res"], ad.obs[_BATCH])
     kbet_per_label = lambda ad, fn: fn(ad.uns["50_neighbor_res"], ad.obs[_BATCH], ad.obs[_LABELS])
@@ -156,6 +157,7 @@ class Benchmarker:
         self._label_key = label_key
         self._n_jobs = n_jobs
         self._progress_bar = progress_bar
+        self._compute_neighbors = True
 
         if self._bio_conservation_metrics is None and self._batch_correction_metrics is None:
             raise ValueError("Either batch or bio metrics must be defined.")
@@ -191,19 +193,25 @@ class Benchmarker:
             self._emb_adatas[emb_key].obsm[_X_PRE] = self._adata.obsm[self._pre_integrated_embedding_obsm_key]
 
         # Compute neighbors
-        progress = self._emb_adatas.values()
-        if self._progress_bar:
-            progress = tqdm(progress, desc="Computing neighbors")
-
-        for ad in progress:
-            if neighbor_computer is not None:
-                neigh_result = neighbor_computer(ad.X, max(self._neighbor_values))
-            else:
-                neigh_result = pynndescent(
-                    ad.X, n_neighbors=max(self._neighbor_values), random_state=0, n_jobs=self._n_jobs
-                )
-            for n in self._neighbor_values:
-                ad.uns[f"{n}_neighbor_res"] = neigh_result.subset_neighbors(n=n)
+        if self._compute_neighbors:
+            progress = self._emb_adatas.values()
+            if self._progress_bar:
+                progress = tqdm(progress, desc="Computing neighbors")
+
+            for ad in progress:
+                if neighbor_computer is not None:
+                    neigh_result = neighbor_computer(ad.X, max(self._neighbor_values))
+                else:
+                    neigh_result = pynndescent(
+                        ad.X, n_neighbors=max(self._neighbor_values), random_state=0, n_jobs=self._n_jobs
+                    )
+                for n in self._neighbor_values:
+                    ad.uns[f"{n}_neighbor_res"] = neigh_result.subset_neighbors(n=n)
+        else:
+            warnings.warn(
+                "Computing Neighbors Skipped",
+                UserWarning,
+            )
 
         self._prepared = True
 
@@ -251,7 +259,7 @@ class Benchmarker:
 
         self._benchmarked = True
 
-    def get_results(self, min_max_scale: bool = True, clean_names: bool = True) -> pd.DataFrame:
+    def get_results(self, min_max_scale: bool = False, clean_names: bool = True) -> pd.DataFrame:
         """Return the benchmarking results.
 
         Parameters
@@ -291,7 +299,7 @@ class Benchmarker:
         df.loc[_METRIC_TYPE, per_class_score.columns] = _AGGREGATE_SCORE
         return df
 
-    def plot_results_table(self, min_max_scale: bool = True, show: bool = True, save_dir: str | None = None) -> Table:
+    def plot_results_table(self, min_max_scale: bool = False, show: bool = True, save_dir: str | None = None) -> Table:
         """Plot the benchmarking results.
 
         Parameters

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/src/scib_metrics/metrics/__init__.py

@@ -4,13 +4,14 @@ from ._kbet import kbet, kbet_per_label
 from ._lisi import clisi_knn, ilisi_knn, lisi_knn
 from ._nmi_ari import nmi_ari_cluster_labels_kmeans, nmi_ari_cluster_labels_leiden
 from ._pcr_comparison import pcr_comparison
-from ._silhouette import silhouette_batch, silhouette_label
+from ._silhouette import bras, silhouette_batch, silhouette_label
 
 __all__ = [
     "isolated_labels",
     "pcr_comparison",
     "silhouette_label",
     "silhouette_batch",
+    "bras",
     "ilisi_knn",
     "clisi_knn",
     "lisi_knn",

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/src/scib_metrics/metrics/_kbet.py

@@ -138,14 +138,8 @@ def kbet_per_label(
     conn_graph = X.knn_graph_connectivities
 
     # prepare call of kBET per cluster
-    clusters = []
-    clusters, counts = np.unique(labels, return_counts=True)
-    skipped = clusters[counts > 10]
-    clusters = clusters[counts <= 10]
-    kbet_scores = {"cluster": list(skipped), "kBET": [np.nan] * len(skipped)}
-    logger.info(f"{len(skipped)} clusters consist of a single batch or are too small. Skip.")
-
-    for clus in clusters:
+    kbet_scores = {"cluster": [], "kBET": []}
+    for clus in np.unique(labels):
         # subset by label
         mask = labels == clus
         conn_graph_sub = conn_graph[mask, :][:, mask]
@@ -153,55 +147,60 @@ def kbet_per_label(
         n_obs = conn_graph_sub.shape[0]
         batches_sub = batches[mask]
 
-        quarter_mean = np.floor(np.mean(pd.Series(batches_sub).value_counts()) / 4).astype("int")
-        k0 = np.min([70, np.max([10, quarter_mean])])
-        # check k0 for reasonability
-        if k0 * n_obs >= size_max:
-            k0 = np.floor(size_max / n_obs).astype("int")
-
-        n_comp, labs = scipy.sparse.csgraph.connected_components(conn_graph_sub, connection="strong")
-
-        if n_comp == 1:  # a single component to compute kBET on
-            try:
-                diffusion_n_comps = np.min([diffusion_n_comps, n_obs - 1])
-                nn_graph_sub = diffusion_nn(conn_graph_sub, k=k0, n_comps=diffusion_n_comps)
-                # call kBET
-                score, _, _ = kbet(
-                    nn_graph_sub,
-                    batches=batches_sub,
-                    alpha=alpha,
-                )
-            except ValueError:
-                logger.info("Diffusion distance failed. Skip.")
-                score = 0  # i.e. 100% rejection
-
+        # check if neighborhood size too small or only one batch in subset
+        if np.logical_or(n_obs < 10, len(np.unique(batches_sub)) == 1):
+            logger.info(f"{clus} consists of a single batch or is too small. Skip.")
+            score = np.nan
         else:
-            # check the number of components where kBET can be computed upon
-            comp_size = pd.Series(labs).value_counts()
-            # check which components are small
-            comp_size_thresh = 3 * k0
-            idx_nonan = np.flatnonzero(np.in1d(labs, comp_size[comp_size >= comp_size_thresh].index))
-
-            # check if 75% of all cells can be used for kBET run
-            if len(idx_nonan) / len(labs) >= 0.75:
-                # create another subset of components, assume they are not visited in a diffusion process
-                conn_graph_sub_sub = conn_graph_sub[idx_nonan, :][:, idx_nonan]
-                conn_graph_sub_sub.sort_indices()
+            quarter_mean = np.floor(np.mean(pd.Series(batches_sub).value_counts()) / 4).astype("int")
+            k0 = np.min([70, np.max([10, quarter_mean])])
+            # check k0 for reasonability
+            if k0 * n_obs >= size_max:
+                k0 = np.floor(size_max / n_obs).astype("int")
+
+            n_comp, labs = scipy.sparse.csgraph.connected_components(conn_graph_sub, connection="strong")
 
+            if n_comp == 1:  # a single component to compute kBET on
                 try:
-                    diffusion_n_comps = np.min([diffusion_n_comps, conn_graph_sub_sub.shape[0] - 1])
-                    nn_results_sub_sub = diffusion_nn(conn_graph_sub_sub, k=k0, n_comps=diffusion_n_comps)
+                    diffusion_n_comps = np.min([diffusion_n_comps, n_obs - 1])
+                    nn_graph_sub = diffusion_nn(conn_graph_sub, k=k0, n_comps=diffusion_n_comps)
                     # call kBET
                     score, _, _ = kbet(
-                        nn_results_sub_sub,
-                        batches=batches_sub[idx_nonan],
+                        nn_graph_sub,
+                        batches=batches_sub,
                         alpha=alpha,
                     )
                 except ValueError:
                     logger.info("Diffusion distance failed. Skip.")
                     score = 0  # i.e. 100% rejection
-            else:  # if there are too many too small connected components, set kBET score to 0
-                score = 0  # i.e. 100% rejection
+
+            else:
+                # check the number of components where kBET can be computed upon
+                comp_size = pd.Series(labs).value_counts()
+                # check which components are small
+                comp_size_thresh = 3 * k0
+                idx_nonan = np.flatnonzero(np.in1d(labs, comp_size[comp_size >= comp_size_thresh].index))
+
+                # check if 75% of all cells can be used for kBET run
+                if len(idx_nonan) / len(labs) >= 0.75:
+                    # create another subset of components, assume they are not visited in a diffusion process
+                    conn_graph_sub_sub = conn_graph_sub[idx_nonan, :][:, idx_nonan]
+                    conn_graph_sub_sub.sort_indices()
+
+                    try:
+                        diffusion_n_comps = np.min([diffusion_n_comps, conn_graph_sub_sub.shape[0] - 1])
+                        nn_results_sub_sub = diffusion_nn(conn_graph_sub_sub, k=k0, n_comps=diffusion_n_comps)
+                        # call kBET
+                        score, _, _ = kbet(
+                            nn_results_sub_sub,
+                            batches=batches_sub[idx_nonan],
+                            alpha=alpha,
+                        )
+                    except ValueError:
+                        logger.info("Diffusion distance failed. Skip.")
+                        score = 0  # i.e. 100% rejection
+                else:  # if there are too many too small connected components, set kBET score to 0
+                    score = 0  # i.e. 100% rejection
 
         kbet_scores["cluster"].append(clus)
         kbet_scores["kBET"].append(score)

scib_metrics-0.5.6/src/scib_metrics/metrics/_silhouette.py

@@ -0,0 +1,162 @@
+from typing import Literal
+
+import numpy as np
+import pandas as pd
+
+from scib_metrics.utils import silhouette_samples
+
+
+def silhouette_label(
+    X: np.ndarray,
+    labels: np.ndarray,
+    rescale: bool = True,
+    chunk_size: int = 256,
+    metric: Literal["euclidean", "cosine"] = "euclidean",
+) -> float:
+    """Average silhouette width (ASW) :cite:p:`luecken2022benchmarking`.
+
+    Default parameters ('euclidean') match scIB implementation.
+
+    Parameters
+    ----------
+    X
+        Array of shape (n_cells, n_features).
+    labels
+        Array of shape (n_cells,) representing label values
+    rescale
+        Scale asw into the range [0, 1].
+    chunk_size
+        Size of chunks to process at a time for distance computation
+    metric
+        The distance metric to use. The distance function can be 'euclidean' (default) or 'cosine'.
+
+    Returns
+    -------
+    silhouette score
+    """
+    asw = np.mean(silhouette_samples(X, labels, chunk_size=chunk_size, metric=metric))
+    if rescale:
+        asw = (asw + 1) / 2
+    return np.mean(asw)
+
+
+def silhouette_batch(
+    X: np.ndarray,
+    labels: np.ndarray,
+    batch: np.ndarray,
+    rescale: bool = True,
+    chunk_size: int = 256,
+    metric: Literal["euclidean", "cosine"] = "euclidean",
+    between_cluster_distances: Literal["nearest", "mean_other", "furthest"] = "nearest",
+) -> float:
+    """Average silhouette width (ASW) with respect to batch ids within each label :cite:p:`luecken2022benchmarking`.
+
+    Default parameters ('euclidean', 'nearest') match scIB implementation.
+
+    Additional options enable BRAS compatible usage (see :func:`~scib_metrics.metrics.bras` documentation).
+
+    Parameters
+    ----------
+    X
+        Array of shape (n_cells, n_features).
+    labels
+        Array of shape (n_cells,) representing label values
+    batch
+        Array of shape (n_cells,) representing batch values
+    rescale
+        Scale asw into the range [0, 1]. If True, higher values are better.
+    chunk_size
+        Size of chunks to process at a time for distance computation.
+    metric
+        The distance metric to use. The distance function can be 'euclidean' (default) or 'cosine'.
+    between_cluster_distances
+        Method for computing inter-cluster distances.
+        - 'nearest': Standard silhouette (distance to nearest cluster)
+        - 'mean_other': BRAS-specific (mean distance to all other clusters)
+        - 'furthest': BRAS-specific (distance to furthest cluster)
+
+    Returns
+    -------
+    silhouette score
+    """
+    sil_dfs = []
+    unique_labels = np.unique(labels)
+    for group in unique_labels:
+        labels_mask = labels == group
+        X_subset = X[labels_mask]
+        batch_subset = batch[labels_mask]
+        n_batches = len(np.unique(batch_subset))
+
+        if (n_batches == 1) or (n_batches == X_subset.shape[0]):
+            continue
+
+        sil_per_group = silhouette_samples(
+            X_subset,
+            batch_subset,
+            chunk_size=chunk_size,
+            metric=metric,
+            between_cluster_distances=between_cluster_distances,
+        )
+
+        # take only absolute value
+        sil_per_group = np.abs(sil_per_group)
+
+        if rescale:
+            # scale s.t. highest number is optimal
+            sil_per_group = 1 - sil_per_group
+
+        sil_dfs.append(
+            pd.DataFrame(
+                {
+                    "group": [group] * len(sil_per_group),
+                    "silhouette_score": sil_per_group,
+                }
+            )
+        )
+
+    sil_df = pd.concat(sil_dfs).reset_index(drop=True)
+    sil_means = sil_df.groupby("group").mean()
+    asw = sil_means["silhouette_score"].mean()
+
+    return asw
+
+
+def bras(
+    X: np.ndarray,
+    labels: np.ndarray,
+    batch: np.ndarray,
+    chunk_size: int = 256,
+    metric: Literal["euclidean", "cosine"] = "cosine",
+    between_cluster_distances: Literal["mean_other", "furthest"] = "mean_other",
+) -> float:
+    """Batch removal adapted silhouette (BRAS) for single-cell data integration assessment :cite:p:`rautenstrauch2025`.
+
+    BRAS evaluates batch effect removal with respect to batch ids within each label (cell type cluster),
+    using a modified silhouette score that accounts for nested batch effects. Unlike standard silhouette,
+    BRAS computes between-cluster distances using the `between_cluster_distances` method rather than
+    nearest-cluster approach. A higher scores indicates better batch mixing.
+
+    Parameters
+    ----------
+    X
+        Array of shape (n_cells, n_features).
+    labels
+        Array of shape (n_cells,) representing label values
+    batch
+        Array of shape (n_cells,) representing batch values
+    rescale
+        Scale asw into the range [0, 1]. If True, higher values are better.
+    chunk_size
+        Size of chunks to process at a time for distance computation.
+    metric
+        The distance metric to use. The distance function can be 'euclidean' (default) or 'cosine'.
+    between_cluster_distances
+        Method for computing inter-cluster distances.
+        - 'mean_other': Mean distance to all cells in other clusters (default)
+        - 'furthest': Distance to furthest cluster (conservative estimate)
+
+    Returns
+    -------
+    BRAS score
+    """
+    return silhouette_batch(X, labels, batch, True, chunk_size, metric, between_cluster_distances)

{scib_metrics-0.5.4 → scib_metrics-0.5.6}/src/scib_metrics/utils/_dist.py

@@ -1,3 +1,6 @@
+from functools import partial
+from typing import Literal
+
 import jax
 import jax.numpy as jnp
 import numpy as np
@@ -10,10 +13,20 @@ def _euclidean_distance(x: np.array, y: np.array) -> float:
 
 
 @jax.jit
-def cdist(x: np.ndarray, y: np.ndarray) -> jnp.ndarray:
+def _cosine_distance(x: np.array, y: np.array) -> float:
+    xy = jnp.dot(x, y)
+    xx = jnp.dot(x, x)
+    yy = jnp.dot(y, y)
+    dist = 1.0 - xy / jnp.sqrt(xx * yy)
+    # Clip the result to avoid rounding error
+    return jnp.clip(dist, 0.0, 2.0)
+
+
+@partial(jax.jit, static_argnames=["metric"])
+def cdist(x: np.ndarray, y: np.ndarray, metric: Literal["euclidean", "cosine"] = "euclidean") -> jnp.ndarray:
     """Jax implementation of :func:`scipy.spatial.distance.cdist`.
 
-    Uses euclidean distance.
+    Uses euclidean distance by default, cosine distance is also available.
 
     Parameters
     ----------
@@ -21,13 +34,20 @@ def cdist(x: np.ndarray, y: np.ndarray) -> jnp.ndarray:
         Array of shape (n_cells_a, n_features)
     y
         Array of shape (n_cells_b, n_features)
+    metric
+        The distance metric to use. The distance function can be 'euclidean' (default) or 'cosine'.
 
     Returns
     -------
     dist
         Array of shape (n_cells_a, n_cells_b)
     """
-    return jax.vmap(lambda x1: jax.vmap(lambda y1: _euclidean_distance(x1, y1))(y))(x)
+    if metric not in ["euclidean", "cosine"]:
+        raise ValueError("Invalid metric choice, must be one of ['euclidean' or 'cosine'].")
+    if metric == "cosine":
+        return jax.vmap(lambda x1: jax.vmap(lambda y1: _cosine_distance(x1, y1))(y))(x)
+    else:
+        return jax.vmap(lambda x1: jax.vmap(lambda y1: _euclidean_distance(x1, y1))(y))(x)
 
 
 @jax.jit