bblean 0.8.0__tar.gz → 0.8.2__tar.gz

{bblean-0.8.0 → bblean-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bblean
-Version: 0.8.0
+Version: 0.8.2
 Summary: BitBirch-Lean Python package
 Author: The Miranda-Quintana Lab and other BitBirch developers
 Author-email: Ramon Alain Miranda Quintana <quintana@chem.ufl.edu>, Krisztina Zsigmond <kzsigmond@ufl.edu>, Ignacio Pickering <ipickering@ufl.edu>, Kenneth Lopez Perez <klopezperez@chem.ufl.edu>, Miroslav Lzicar <miroslav.lzicar@deepmedchem.com>

{bblean-0.8.0 → bblean-0.8.2}/bblean/_merges.py

@@ -69,6 +69,48 @@ class DiameterMerge(MergeAcceptFunction):
         return jt_isim_from_sum(new_ls, new_n) >= threshold
 
 
+class FlexibleToleranceDiameterMerge(MergeAcceptFunction):
+    name = "flexible-tolerance-diameter"
+    # NOTE: Equivalent to tolerance-diameter but uses min(old_dc, threshold) as the
+    # criteria
+
+    def __init__(
+        self,
+        tolerance: float = 0.05,
+        n_max: int = 1000,
+        decay: float = 1e-3,
+        adaptive: bool = True,
+    ) -> None:
+        self.tolerance = tolerance
+        self.decay = decay
+        self.offset = np.exp(-decay * n_max)
+        if not adaptive:
+            self.decay = 0.0
+            self.offset = 0.0
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        new_dc = jt_isim_from_sum(new_ls, new_n)
+        if new_dc < threshold:
+            return False
+        if old_n == 1:
+            return True
+        old_dc = jt_isim_from_sum(old_ls, old_n)
+        tol = max(self.tolerance * (np.exp(-self.decay * old_n) - self.offset), 0.0)
+        return new_dc >= min(old_dc, threshold) - tol
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.tolerance})"
+
+
 class ToleranceDiameterMerge(MergeAcceptFunction):
     name = "tolerance-diameter"
     # NOTE: The reliability of the estimate of the cluster should be a function of the
@@ -202,6 +244,8 @@ def get_merge_accept_fn(
         return ToleranceMerge(tolerance)
     elif merge_criterion == "tolerance-diameter":
         return ToleranceDiameterMerge(tolerance)
+    elif merge_criterion == "flexible-tolerance-diameter":
+        return FlexibleToleranceDiameterMerge(tolerance)
     elif merge_criterion == "tolerance-radius":
         return ToleranceRadiusMerge(tolerance)
     elif merge_criterion == "never-merge":

{bblean-0.8.0 → bblean-0.8.2}/bblean/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.8.0'
-__version_tuple__ = version_tuple = (0, 8, 0)
+__version__ = version = '0.8.2'
+__version_tuple__ = version_tuple = (0, 8, 2)
 
 __commit_id__ = commit_id = None

{bblean-0.8.0 → bblean-0.8.2}/bblean/bitbirch.py

@@ -75,6 +75,8 @@ from bblean.similarity import (
     jt_most_dissimilar_packed,
     jt_isim_medoid,
     centroid_from_sum,
+    estimate_jt_std,
+    jt_isim,
 )
 
 if os.getenv("BITBIRCH_NO_EXTENSIONS"):
@@ -90,6 +92,64 @@ else:
 __all__ = ["BitBirch"]
 
 
+@tp.overload
+def guess_threshold(
+    fps: NDArray[np.uint8],
+    input_is_packed: bool = True,
+    n_features: int | None = None,
+    max_samples: int = 1_000_000,
+    factor: float = 3.0,
+    return_mean_std: tp.Literal[False] = False,
+) -> float:
+    pass
+
+
+@tp.overload
+def guess_threshold(
+    fps: NDArray[np.uint8],
+    input_is_packed: bool = True,
+    n_features: int | None = None,
+    max_samples: int = 1_000_000,
+    factor: float = 3.0,
+    return_mean_std: tp.Literal[True] = True,
+) -> tuple[float, float, float]:
+    pass
+
+
+def guess_threshold(
+    fps: NDArray[np.uint8],
+    input_is_packed: bool = True,
+    n_features: int | None = None,
+    max_samples: int = 1_000_000,
+    factor: float = 3.0,
+    return_mean_std: bool = False,
+) -> float | tuple[float, float, float]:
+    r""":meta private:
+
+    Guess the optimal bitbirch threshold
+
+    Uses the heuristic mean_tanimoto + 3.0 * std_tanimoto
+    """
+    num_fps = len(fps)
+    if num_fps > max_samples:
+        rng = np.random.default_rng(42)
+        random_choices = rng.choice(num_fps, size=max_samples, replace=False)
+        fps = fps[random_choices]
+        num_fps = len(fps)
+    mean = jt_isim(fps, input_is_packed, n_features)
+    if num_fps <= 50:
+        n_samples = num_fps
+    else:
+        n_samples = max(5 * np.sqrt(num_fps), 50)
+    std = estimate_jt_std(
+        fps, input_is_packed=input_is_packed, n_features=n_features, n_samples=n_samples
+    )
+    thresh = mean + factor * std
+    if return_mean_std:
+        return thresh, mean, std
+    return thresh
+
+
 # For backwards compatibility with the global "set_merge", keep weak references to all
 # the BitBirch instances and update them when set_merge is called
 _BITBIRCH_INSTANCES: WeakSet["BitBirch"] = WeakSet()

{bblean-0.8.0 → bblean-0.8.2}/bblean/cli.py

@@ -1,5 +1,6 @@
 r"""Command line interface entrypoints"""
 
+import numpy as np
 import warnings
 import random
 import typing as tp
@@ -930,6 +931,54 @@ def _plot_summary(
         )
 
 
+@app.command("thresh")
+def _guess_threshold(
+    ctx: Context,
+    input_: Annotated[
+        Path,
+        Argument(help="`*.npy` file with fingerprints"),
+    ],
+    factor: Annotated[
+        float,
+        Option("-f", "--factor"),
+    ] = 3.0,
+    n_features: Annotated[
+        int | None,
+        Option(
+            "--n-features",
+            help="Number of features in the fingerprints."
+            " It must be provided for packed inputs *if it is not a multiple of 8*."
+            " For typical fingerprint sizes (e.g. 2048, 1024), it is not required",
+            rich_help_panel="Advanced",
+        ),
+    ] = None,
+    input_is_packed: Annotated[
+        bool,
+        Option(
+            "--packed-input/--unpacked-input",
+            help="Toggle whether the input consists on packed or unpacked fingerprints",
+            rich_help_panel="Advanced",
+        ),
+    ] = True,
+    max_samples: Annotated[
+        int,
+        Option("-m", "--max-samples"),
+    ] = 1_000_000,
+) -> None:
+    r"""Estimate the optimal BitBirch threshold for a fingerprints file"""
+    from bblean.bitbirch import guess_threshold
+    from bblean._console import get_console
+
+    console = get_console()
+    fps = np.load(input_)
+    thresh, mean, std = guess_threshold(
+        fps, input_is_packed, n_features, max_samples, factor, return_mean_std=True
+    )
+    console.print(f"Estimated average similarity: {mean:.4f}")
+    console.print(f"Estimated similarity deviation: {std:.4f}")
+    console.print(f"Estimated optimal threshold: {thresh:.4f}")
+
+
 @app.command("run")
 def _run(
     ctx: Context,

{bblean-0.8.0 → bblean-0.8.2}/bblean/fingerprints.py

@@ -1,11 +1,15 @@
 r"""Utilites for manipulating fingerprints and fingerprint files"""
 
+import sys
+import math
+import weakref
 import warnings
 import dataclasses
 from pathlib import Path
 from numpy.typing import NDArray, DTypeLike
 import numpy as np
 import typing as tp
+import multiprocessing as mp
 import multiprocessing.shared_memory as shmem
 
 from rich.console import Console
@@ -13,6 +17,8 @@ from rdkit.Chem import rdFingerprintGenerator, MolFromSmiles, SanitizeFlags, San
 
 from bblean._config import DEFAULTS
 from bblean._console import get_console
+from bblean.smiles import _iter_ranges_and_smiles_batches
+from bblean.utils import _num_avail_cpus
 
 __all__ = [
     "make_fake_fingerprints",
@@ -441,3 +447,112 @@ class _FingerprintArrayFiller:
             fps[i, :] = fp
         fps_shmem.close()
         invalid_mask_shmem.close()
+
+
+@tp.overload
+def fps_from_smiles_parallel(
+    smiles: tp.Iterable[str],
+    kind: str = DEFAULTS.fp_kind,
+    n_features: int = DEFAULTS.n_features,
+    dtype: DTypeLike = np.uint8,
+    sanitize: str = "all",
+    skip_invalid: tp.Literal[False] = False,
+    pack: bool = True,
+    num_ps: int = 1,
+    replace_dummy_atoms: bool = False,
+    tab_separated: bool = False,
+    mp_context: tp.Any = None,
+) -> NDArray[np.uint8]:
+    pass
+
+
+@tp.overload
+def fps_from_smiles_parallel(
+    smiles: tp.Iterable[str],
+    kind: str = DEFAULTS.fp_kind,
+    n_features: int = DEFAULTS.n_features,
+    dtype: DTypeLike = np.uint8,
+    sanitize: str = "all",
+    skip_invalid: tp.Literal[True] = True,
+    pack: bool = True,
+    num_ps: int = 1,
+    replace_dummy_atoms: bool = False,
+    tab_separated: bool = False,
+    mp_context: tp.Any = None,
+) -> tp.Union[NDArray[np.uint8], tuple[NDArray[np.uint8], NDArray[np.int64]]]:
+    pass
+
+
+# NOTE: This function is proof of concept and kinda dangerous since it registers
+# a custom destructor for the numpy array
+# It is also *only usable if called inside an if __name__ == "__main__" guard*
+# For now lets hide it
+def fps_from_smiles_parallel(
+    smiles: tp.Iterable[str],
+    kind: str = DEFAULTS.fp_kind,
+    n_features: int = DEFAULTS.n_features,
+    dtype: DTypeLike = np.uint8,
+    sanitize: str = "all",
+    skip_invalid: bool = False,
+    pack: bool = True,
+    num_ps: int | None = None,
+    replace_dummy_atoms: bool = False,
+    tab_separated: bool = False,
+    mp_context: tp.Any = None,
+) -> tp.Union[NDArray[np.uint8], tuple[NDArray[np.uint8], NDArray[np.int64]]]:
+    r""":meta private:"""
+    if mp_context is None:
+        mp_context = mp.get_context("forkserver" if sys.platform == "linux" else None)
+    if isinstance(smiles, str):
+        smiles = [smiles]
+    smiles = list(smiles)
+    smiles_num = len(smiles)
+    if num_ps is None:
+        num_ps = _num_avail_cpus()
+
+    if pack:
+        out_dim = (n_features + 7) // 8
+    else:
+        out_dim = n_features
+    shmem_size = smiles_num * out_dim * np.dtype(dtype).itemsize
+    fps_shmem = shmem.SharedMemory(create=True, size=shmem_size)
+    invalid_mask_shmem = shmem.SharedMemory(create=True, size=smiles_num)
+    fps_array_filler = _FingerprintArrayFiller(
+        shmem_name=fps_shmem.name,
+        invalid_mask_shmem_name=invalid_mask_shmem.name,
+        kind=kind,
+        fp_size=n_features,
+        num_smiles=smiles_num,
+        dtype=np.dtype(dtype).name,
+        pack=pack,
+        sanitize=sanitize,
+        skip_invalid=skip_invalid,
+    )
+    num_per_batch = math.ceil(smiles_num / num_ps)
+    with mp_context.Pool(processes=num_ps) as pool:
+        pool.starmap(
+            fps_array_filler,
+            _iter_ranges_and_smiles_batches(
+                smiles,
+                num_per_batch,
+                tab_separated,
+                replace_dummy_atoms,
+                assume_paths=False,
+            ),
+        )
+    fps = np.ndarray((smiles_num, out_dim), dtype=dtype, buffer=fps_shmem.buf)
+    mask = np.ndarray((smiles_num,), dtype=np.bool, buffer=invalid_mask_shmem.buf)
+    if skip_invalid:
+        fps = np.delete(fps, mask, axis=0)
+        weakref.finalize(mask, invalid_mask_shmem.close)
+        weakref.finalize(mask, invalid_mask_shmem.unlink)
+        weakref.finalize(fps, fps_shmem.close)
+        weakref.finalize(fps, fps_shmem.unlink)
+        return fps, mask
+
+    del mask
+    invalid_mask_shmem.close()
+    invalid_mask_shmem.unlink()
+    weakref.finalize(fps, fps_shmem.close)
+    weakref.finalize(fps, fps_shmem.unlink)
+    return fps

{bblean-0.8.0 → bblean-0.8.2}/bblean/similarity.py

@@ -293,7 +293,7 @@ def estimate_jt_std(
     n_samples: int | None = None,
     input_is_packed: bool = True,
     n_features: int | None = None,
-    min_samples: int = 1_000_000,
+    max_samples: int = 1_000_000,
 ) -> float:
     r"""Estimate the std of all pairwise Tanimoto.
 
@@ -303,15 +303,19 @@ def estimate_jt_std(
         The standard deviation of all pairwise Tanimoto among the sampled fingerprints.
     """
     num_fps = len(fps)
-    if num_fps > min_samples:
-        np.random.seed(42)
-        random_choices = np.random.choice(num_fps, size=min_samples, replace=False)
+    if num_fps > max_samples:
+        rng = np.random.default_rng(42)
+        random_choices = rng.choice(num_fps, size=max_samples, replace=False)
         fps = fps[random_choices]
         num_fps = len(fps)
     if n_samples is None:
         # Heuristic: use at least 50 samples, or 1 per 10,000 fingerprints,
         # to balance statistical representativeness and computational efficiency
-        n_samples = max(num_fps // 10_000, 50)
+        # TODO: This heuristic is broken, too few samples until 500k
+        if num_fps <= 500_000:
+            n_samples = 50
+        else:
+            n_samples = num_fps // 10_000
     sample_idxs = jt_stratified_sampling(fps, n_samples, input_is_packed, n_features)
 
     # Work with only the sampled fingerprints

{bblean-0.8.0 → bblean-0.8.2}/bblean/smiles.py

@@ -71,12 +71,14 @@ def _iter_ranges_and_smiles_batches(
     num_per_batch: int,
     tab_separated: bool = False,
     replace_dummy_atoms: bool = False,
+    assume_paths: bool = True,
 ) -> tp.Iterable[tuple[tuple[int, int], tuple[str, ...]]]:
+    if assume_paths:
+        it = iter_smiles_from_paths(smiles_paths, tab_separated, replace_dummy_atoms)
+    else:
+        it = tp.cast(tp.Iterator[str], smiles_paths)
     start_idx = 0
-    for batch in batched(
-        iter_smiles_from_paths(smiles_paths, tab_separated, replace_dummy_atoms),
-        num_per_batch,
-    ):
+    for batch in batched(it, num_per_batch):
         size = len(batch)
         end_idx = start_idx + size
         yield (start_idx, end_idx), batch

{bblean-0.8.0 → bblean-0.8.2}/bblean.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bblean
-Version: 0.8.0
+Version: 0.8.2
 Summary: BitBirch-Lean Python package
 Author: The Miranda-Quintana Lab and other BitBirch developers
 Author-email: Ramon Alain Miranda Quintana <quintana@chem.ufl.edu>, Krisztina Zsigmond <kzsigmond@ufl.edu>, Ignacio Pickering <ipickering@ufl.edu>, Kenneth Lopez Perez <klopezperez@chem.ufl.edu>, Miroslav Lzicar <miroslav.lzicar@deepmedchem.com>

{bblean-0.8.0 → bblean-0.8.2}/tests/test_simple.py

@@ -2,7 +2,7 @@ import itertools
 import pytest
 import numpy as np
 
-from bblean.bitbirch import BitBirch
+from bblean.bitbirch import BitBirch, guess_threshold
 from bblean.fingerprints import pack_fingerprints, make_fake_fingerprints
 
 from inline_snapshot import snapshot
@@ -42,6 +42,20 @@ def test_bb_cluster_simple_repeated_fps() -> None:
         assert ids == [list(range(repeats))]
 
 
+def test_guess_threhsold() -> None:
+    fps = make_fake_fingerprints(
+        100, n_features=8, seed=12620509540149709235, pack=True
+    )
+    thresh = guess_threshold(fps, return_mean_std=False)
+    assert thresh > 0.9 and thresh < 1.0
+
+    fps = make_fake_fingerprints(
+        100, n_features=2048, seed=12620509540149709235, pack=True
+    )
+    thresh = guess_threshold(fps, return_mean_std=False)
+    assert thresh > 0.4 and thresh < 0.6
+
+
 def test_bb_cluster_3_fps() -> None:
     fps = make_fake_fingerprints(3, n_features=8, seed=12620509540149709235, pack=True)