bblean 0.6.0b2__cp312-cp312-win_amd64.whl

bblean/_memory.py

@@ -0,0 +1,198 @@
+r"""Monitor and collect memory stats"""
+
+import typing as tp
+import mmap
+import warnings
+from enum import Enum
+import ctypes
+import dataclasses
+from pathlib import Path
+import sys
+import time
+import os
+import multiprocessing as mp
+
+import typing_extensions as tpx
+import psutil
+import numpy as np
+from numpy.typing import NDArray
+from rich.console import Console
+
+_BYTES_TO_GIB = 1 / 1024**3
+
+
+class Madv(Enum):
+    WILLNEED = 3
+    SEQUENTIAL = 2
+    # PAGEOUT and DONTNEED reduce memory usage around 40%
+    # TODO: Check exactly what DONTNEED does. I believe PAGEOUT *swaps out*
+    # so DONTNEED may be preferred since it may have less perf. penalty
+    DONTNEED = 4
+    PAGEOUT = 21
+    FREE = 8  # *ONLY* works on anonymous pages (not file-backed like numpy arrays)
+    # Cold does *not* immediatly release memory, it is just a soft hint that
+    # those pages won't be needed soon
+    COLD = 20
+
+
+# Get handle to the system's libc
+def _get_libc() -> tp.Any:
+    if sys.platform == "linux":
+        return ctypes.CDLL("libc.so.6", use_errno=True)
+    elif sys.platform == "darwin":
+        return ctypes.CDLL("libc.dylib", use_errno=True)
+    # For now, do nothing in Windows
+    return
+
+
+# This reduces memory usage around 40%, since the kernel can release
+# pages once the array has been iterated over. The issue is, after this has been done,
+# the array is out of the RAM, so refinement is not possible.
+def _madvise_dontneed(page_start: int, size: int) -> None:
+    _madvise(page_start, size, Madv.DONTNEED)
+
+
+# let the kernel know that access to this range of addrs will be sequential
+# (pages can be read-ahead and discarded fast after read if needed)
+def _madvise_sequential(page_start: int, size: int) -> None:
+    _madvise(page_start, size, Madv.SEQUENTIAL)
+
+
+def _madvise(page_start: int, size: int, opt: Madv) -> None:
+    libc = _get_libc()
+    if libc is None:
+        return
+    if libc.madvise(ctypes.c_void_p(page_start), size, opt.value) != 0:
+        errno = ctypes.get_errno()
+        warnings.warn(f"{opt} failed with error code {errno}")
+
+
+_Input = tp.Union[NDArray[np.integer], list[NDArray[np.integer]]]
+
+
+@dataclasses.dataclass
+class _ArrayMemPagesManager:
+    can_release: bool
+    _pagesizex: int
+    _iters_per_pagex: int
+    _curr_page_start_addr: int
+
+    @classmethod
+    def from_bb_input(cls, X: _Input, can_release: bool | None = None) -> tpx.Self:
+        pagesizex = mmap.PAGESIZE * 512
+        if (
+            isinstance(X, np.memmap)
+            and X.ndim == 2
+            and (pagesizex % X.shape[1] == 0)
+            and X.offset < X.shape[1]
+        ):
+            # In most cases pagesizex % n_features == 0 and offset < n_features
+            # Every n_iters, release the prev page and add pagesizex to start_addr
+            iters_per_pagex = int(pagesizex / X.shape[1])  # ~ 8192 iterations
+            curr_page_start_addr = X.ctypes.data - X.offset
+            _can_release = True
+        else:
+            iters_per_pagex = 0
+            curr_page_start_addr = 0
+            _can_release = False
+        if can_release is not None:
+            _can_release = can_release
+        return cls(_can_release, pagesizex, iters_per_pagex, curr_page_start_addr)
+
+    def should_release_curr_page(self, row_idx: int) -> bool:
+        return row_idx % self._iters_per_pagex == 0
+
+    def release_curr_page_and_update_addr(self) -> None:
+        _madvise_dontneed(self._curr_page_start_addr, self._pagesizex)
+        self._curr_page_start_addr += self._pagesizex
+
+
+def _mmap_file_and_madvise_sequential(
+    path: Path, max_fps: int | None = None
+) -> NDArray[np.integer]:
+    arr = np.load(path, mmap_mode="r")[:max_fps]
+    # Numpy actually puts the *whole file* in mmap mode (arr + header)
+    # This means the array data starts from a nonzero offset starting from the backing
+    # buffer if we want the address to the start of the file we need to displace the
+    # addr of the arry by the bsize of the header, which can be accessed by arr.offset
+    #
+    # This is required since madvise needs a page-aligned address (address must
+    # be a multiple of mmap.PAGESIZE (portable) == os.sysconf("SC_PAGE_SIZE")
+    # (mac|linux), typically 4096 B).
+    #
+    # TODO: In some cases, for some reason, this fails with errno 22
+    # failure is harmless, but could incurr in a slight perf penalty
+    _madvise_sequential(arr.ctypes.data - arr.offset, arr.nbytes)
+    return arr
+
+
+def system_mem_gib() -> tuple[int, int] | tuple[None, None]:
+    mem = psutil.virtual_memory()
+    return mem.total * _BYTES_TO_GIB, mem.available * _BYTES_TO_GIB
+
+
+def get_peak_memory_gib(out_dir: Path) -> float | None:
+    file = out_dir / "max-rss.txt"
+    if not file.exists():
+        return None
+    with open(file, mode="r", encoding="utf-8") as f:
+        peak_mem_gib = float(f.read().strip())
+    return peak_mem_gib
+
+
+def monitor_rss_process(
+    file: Path | str, interval_s: float, start_time: float, parent_pid: int
+) -> None:
+    file = Path(file)
+    this_pid = os.getpid()
+    ps = psutil.Process(parent_pid)
+
+    def total_rss() -> float:
+        total_rss = ps.memory_info().rss
+        for proc in ps.children(recursive=True):
+            if proc.pid == this_pid:
+                continue
+            try:
+                total_rss += proc.memory_info().rss
+            except psutil.NoSuchProcess:
+                # Prevent race condition since process may have finished before it can
+                # be polled
+                continue
+        return total_rss
+
+    with open(file, mode="w", encoding="utf-8") as f:
+        f.write("rss_gib,time_s\n")
+        f.flush()
+        os.fsync(f.fileno())
+
+    max_rss_gib = 0.0
+    while True:
+        total_rss_gib = total_rss() * _BYTES_TO_GIB
+        with open(file, mode="a", encoding="utf-8") as f:
+            f.write(f"{total_rss_gib},{time.perf_counter() - start_time}\n")
+            f.flush()
+            os.fsync(f.fileno())
+        if total_rss_gib > max_rss_gib:
+            max_rss_gib = total_rss_gib
+            with open(file.parent / "max-rss.txt", mode="w", encoding="utf-8") as f:
+                f.write(f"{max_rss_gib}\n")
+                f.flush()
+                os.fsync(f.fileno())
+        time.sleep(interval_s)
+
+
+def launch_monitor_rss_daemon(
+    out_file: Path, interval_s: float, console: Console | None = None
+) -> None:
+    if console is not None:
+        console.print("** Monitoring total RAM usage **\n")
+    mp.Process(
+        target=monitor_rss_process,
+        kwargs=dict(
+            file=out_file,
+            interval_s=interval_s,
+            start_time=time.perf_counter(),
+            parent_pid=os.getpid(),
+        ),
+        daemon=True,
+    ).start()

bblean/_merges.py

@@ -0,0 +1,212 @@
+r"""Merging criteria for BitBIRCH clustering"""
+
+import numpy as np
+from numpy.typing import NDArray
+
+# NOTE: jt_isim_from_sum is equivalent to jt_isim_diameter_compl_from_sum
+from bblean.similarity import jt_isim_from_sum, jt_isim_radius_compl_from_sum
+
+BUILTIN_MERGES = [
+    "radius",
+    "diameter",
+    "tolerance-diameter",
+    "tolerance-radius",
+    "tolerance-legacy",
+    "never-merge",
+]
+
+
+class MergeAcceptFunction:
+    # For the merge functions, although outputs of jt_isim_from_sum f64, directly using
+    # f64 is *not* faster than starting with uint64
+    name: str = ""
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        raise NotImplementedError("Must be implemented by subclasses")
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"
+
+
+class RadiusMerge(MergeAcceptFunction):
+    name = "radius"
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        return jt_isim_radius_compl_from_sum(new_ls, new_n) >= threshold
+
+
+class DiameterMerge(MergeAcceptFunction):
+    name = "diameter"
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        return jt_isim_from_sum(new_ls, new_n) >= threshold
+
+
+class ToleranceDiameterMerge(MergeAcceptFunction):
+    name = "tolerance-diameter"
+    # NOTE: The reliability of the estimate of the cluster should be a function of the
+    # size of the old cluster, so in this metric, tolerance is larger for small clusters
+    # tolerance = max{ alpha * (exp(-decay * N_old) - offset), 0}
+
+    def __init__(
+        self,
+        tolerance: float = 0.05,
+        n_max: int = 1000,
+        decay: float = 1e-3,
+        adaptive: bool = True,
+    ) -> None:
+        self.tolerance = tolerance
+        self.decay = decay
+        self.offset = np.exp(-decay * n_max)
+        if not adaptive:
+            self.decay = 0.0
+            self.offset = 0.0
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        new_dc = jt_isim_from_sum(new_ls, new_n)
+        if new_dc < threshold:
+            return False
+        # If the old n is 1 then merge directly (infinite tolerance), since the
+        # old_d is undefined for a single fp
+        if old_n == 1:
+            return True
+        # Only merge if the new_dc is greater or equal to the old, up to some tolerance,
+        # which decays with N
+        old_dc = jt_isim_from_sum(old_ls, old_n)
+        tol = max(self.tolerance * (np.exp(-self.decay * old_n) - self.offset), 0.0)
+        return new_dc >= old_dc - tol
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.tolerance})"
+
+
+class ToleranceRadiusMerge(ToleranceDiameterMerge):
+    name = "tolerance-radius"
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        new_rc = jt_isim_radius_compl_from_sum(new_ls, new_n)
+        if new_rc < threshold:
+            return False
+        if old_n == 1:
+            return True
+        old_rc = jt_isim_radius_compl_from_sum(old_ls, old_n)
+        tol = max(self.tolerance * (np.exp(-self.decay * old_n) - self.offset), 0.0)
+        return new_rc >= old_rc - tol
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.tolerance})"
+
+
+class NeverMerge(ToleranceDiameterMerge):
+    name = "never-merge"
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        return False
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"
+
+
+class ToleranceMerge(MergeAcceptFunction):
+    name = "tolerance-legacy"
+
+    def __init__(self, tolerance: float = 0.05) -> None:
+        self.tolerance = tolerance
+
+    def __call__(
+        self,
+        threshold: float,
+        new_ls: NDArray[np.integer],
+        new_n: int,
+        old_ls: NDArray[np.integer],
+        nom_ls: NDArray[np.integer],
+        old_n: int,
+        nom_n: int,
+    ) -> bool:
+        # First two branches are equivalent to 'diameter'
+        new_dc = jt_isim_from_sum(new_ls, new_n)
+        if new_dc < threshold:
+            return False
+        if old_n == 1 or nom_n != 1:
+            return True
+        # 'new_dc >= threshold' and 'new_n == old_n + 1' are guaranteed here
+        old_dc = jt_isim_from_sum(old_ls, old_n)
+        return (new_dc * new_n - old_dc * (old_n - 1)) / 2 >= old_dc - self.tolerance
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}({self.tolerance})"
+
+
+def get_merge_accept_fn(
+    merge_criterion: str, tolerance: float = 0.05
+) -> MergeAcceptFunction:
+    if merge_criterion == "radius":
+        return RadiusMerge()
+    elif merge_criterion == "diameter":
+        return DiameterMerge()
+    elif merge_criterion == "tolerance-legacy":
+        return ToleranceMerge(tolerance)
+    elif merge_criterion == "tolerance-diameter":
+        return ToleranceDiameterMerge(tolerance)
+    elif merge_criterion == "tolerance-radius":
+        return ToleranceRadiusMerge(tolerance)
+    elif merge_criterion == "never-merge":
+        return NeverMerge(tolerance)
+    raise ValueError(
+        f"Unknown merge criterion {merge_criterion} "
+        "Valid criteria are: radius|diameter|tolerance-diameter|tolerance-radius"
+    )

bblean/_py_similarity.py

@@ -0,0 +1,278 @@
+r"""Fallback python implementation of molecular similarity calculators"""
+
+import warnings
+
+from numpy.typing import NDArray
+import numpy as np
+
+from bblean.utils import min_safe_uint
+from bblean.fingerprints import unpack_fingerprints, pack_fingerprints
+
+
+def centroid_from_sum(
+    linear_sum: NDArray[np.integer], n_samples: int, *, pack: bool = True
+) -> NDArray[np.uint8]:
+    r"""Calculates the majority vote centroid from a sum of fingerprint values
+
+    The majority vote centroid is an good approximation of the Tanimoto centroid.
+
+    Parameters
+    ----------
+
+    linear_sum : np.ndarray
+                 Sum of the elements column-wise
+    n_samples : int
+                Number of samples
+    pack : bool
+        Whether to pack the resulting fingerprints
+
+    Returns
+    -------
+    centroid : np.ndarray[np.uint8]
+               Centroid fingerprints of the given set
+    """
+    # NOTE: Numpy guarantees bools are stored as 0xFF -> True and 0x00 -> False,
+    # so this view is fully safe
+    if n_samples <= 1:
+        centroid = linear_sum.astype(np.uint8, copy=False)
+    else:
+        centroid = (linear_sum >= n_samples * 0.5).view(np.uint8)
+    if pack:
+        return np.packbits(centroid, axis=-1)
+    return centroid
+
+
+def centroid(
+    fps: NDArray[np.uint8],
+    input_is_packed: bool = True,
+    n_features: int | None = None,
+    *,
+    pack: bool = True,
+) -> NDArray[np.uint8]:
+    r"""Calculates the majority vote centroid from a set of fingerprints
+
+    The majority vote centroid is an good approximation of the Tanimoto centroid.
+    """
+    if input_is_packed:
+        fps = unpack_fingerprints(fps, n_features)
+    return centroid_from_sum(
+        np.sum(fps, axis=0, dtype=np.uint64),  # type: ignore
+        len(fps),
+        pack=pack,
+    )
+
+
+def jt_compl_isim(
+    fps: NDArray[np.uint8], input_is_packed: bool = True, n_features: int | None = None
+) -> NDArray[np.float64]:
+    """Get all complementary (Tanimoto) similarities of a set of fps, using iSIM"""
+    if input_is_packed:
+        fps = unpack_fingerprints(fps, n_features)
+    # Vectorized calculation of all compl isim
+    # For all compl isim N is N_total - 1
+    n_objects = len(fps) - 1
+    if n_objects < 2:
+        msg = "Invalid fps. len(fps) must be >= 3"
+        warnings.warn(msg, RuntimeWarning, stacklevel=2)
+        return np.full(len(fps), fill_value=np.nan, dtype=np.float64)
+    linear_sum = np.sum(fps, axis=0)
+    n_objects = len(fps) - 1
+    comp_sims = [jt_isim_from_sum(linear_sum - fp, n_objects) for fp in fps]
+
+    return np.array(comp_sims, dtype=np.float64)
+
+
+def _jt_isim_medoid_index(
+    fps: NDArray[np.uint8], input_is_packed: bool = True, n_features: int | None = None
+) -> int:
+    return np.argmin(jt_compl_isim(fps, input_is_packed, n_features)).item()
+
+
+def jt_isim_medoid(
+    fps: NDArray[np.uint8],
+    input_is_packed: bool = True,
+    n_features: int | None = None,
+    pack: bool = True,
+) -> tuple[int, NDArray[np.uint8]]:
+    r"""Calculate the (Tanimoto) medoid of a set of fingerprints, using iSIM
+
+    Returns both the index of the medoid in the input array and the medoid itself
+
+    .. note::
+        Returns the first (or only) fingerprint for array of size 2 and 1 respectively.
+        Raises ValueError for arrays of size 0
+
+    """
+    if not fps.size:
+        raise ValueError("Size of fingerprints set must be > 0")
+    if input_is_packed:
+        fps = unpack_fingerprints(fps, n_features)
+    if len(fps) < 3:
+        idx = 0  # Medoid undefined for sets of 3 or more fingerprints
+    else:
+        idx = _jt_isim_medoid_index(fps, input_is_packed=False)
+    m = fps[idx]
+    if pack:
+        return idx, pack_fingerprints(m)
+    return idx, m
+
+
+# Requires numpy >= 2.0
+def _popcount(a: NDArray[np.uint8]) -> NDArray[np.uint32]:
+    # a is packed uint8 array with last axis = bytes
+    # Sum bit-counts across bytes to get per-object totals
+
+    # If the array has columns that are a multiple of 8, doing a bitwise count
+    # over the buffer reinterpreted as uint64 is slightly faster.
+    # This is zero cost if the exception is not triggered. Not having a be a multiple of
+    # 8 is a very unlikely scenario, since fps are typically 1024 or 2048
+    b: NDArray[np.integer]
+    try:
+        b = a.view(np.uint64)
+    except ValueError:
+        b = a
+    return np.bitwise_count(b).sum(axis=-1, dtype=np.uint32)
+
+
+# O(N) approximation to obtain "most dissimilar fingerprints" within an array
+def jt_most_dissimilar_packed(
+    Y: NDArray[np.uint8], n_features: int | None = None
+) -> tuple[np.integer, np.integer, NDArray[np.float64], NDArray[np.float64]]:
+    """Finds two fps in a packed fp array that are the most Tanimoto-dissimilar
+
+    This is not guaranteed to find the most dissimilar fps, it is
+    a robust O(N) approximation that doesn't affect final cluster quality.
+    First find centroid of Y, then find fp_1, the most dissimilar molecule
+    to the centroid. Finally find fp_2, the most dissimilar molecule to fp_1
+
+    Returns
+    -------
+    fp_1 : int
+        index of the first fingerprint
+    fp_2 : int
+        index of the second fingerprint
+    sims_fp_1 : np.ndarray
+        Tanimoto similarities of Y to fp_1
+    sims_fp_2: np.ndarray
+        Tanimoto similarities of Y to fp_2
+    """
+    # Get the centroid of the fps
+    n_samples = len(Y)
+    Y_unpacked = unpack_fingerprints(Y, n_features)
+    # np.sum() automatically promotes to uint64 unless forced to a smaller dtype
+    linear_sum = np.sum(Y_unpacked, axis=0, dtype=min_safe_uint(n_samples))
+    packed_centroid = centroid_from_sum(linear_sum, n_samples, pack=True)
+
+    cardinalities = _popcount(Y)
+
+    # Get similarity of each fp to the centroid, and the least similar fp idx (fp_1)
+    sims_cent = _jt_sim_packed_precalc_cardinalities(Y, packed_centroid, cardinalities)
+    fp_1 = np.argmin(sims_cent)
+
+    # Get similarity of each fp to fp_1, and the least similar fp idx (fp_2)
+    sims_fp_1 = _jt_sim_packed_precalc_cardinalities(Y, Y[fp_1], cardinalities)
+    fp_2 = np.argmin(sims_fp_1)
+
+    # Get similarity of each fp to fp_2
+    sims_fp_2 = _jt_sim_packed_precalc_cardinalities(Y, Y[fp_2], cardinalities)
+    return fp_1, fp_2, sims_fp_1, sims_fp_2
+
+
+def _jt_sim_arr_vec_packed(
+    x: NDArray[np.uint8],
+    y: NDArray[np.uint8],
+) -> NDArray[np.float64]:
+    r"""Tanimoto similarity between packed fingerprints
+
+    Either both inputs are vectors of shape (F,) (Numpy scalar is returned), or one is
+    an vector (F,) and the other an array of shape (N, F) (Numpy array of shape (N,) is
+    returned).
+    """
+    if x.ndim != 2 or y.ndim != 1:
+        raise ValueError("Expected a 2D array and a 1D vector as inputs")
+    return _jt_sim_packed_precalc_cardinalities(x, y, _popcount(x))
+
+
+def _jt_sim_packed_precalc_cardinalities(
+    x: NDArray[np.uint8],
+    y: NDArray[np.uint8],
+    cardinalities: NDArray[np.integer],
+) -> NDArray[np.float64]:
+    # _cardinalities must be the result of calling _popcount(arr)
+
+    # Maximum value in the denominator sum is the 2 * n_features (which is typically
+    # uint16, but we use uint32 for safety)
+    intersection = _popcount(np.bitwise_and(x, y))
+
+    # Return value requires an out-of-place operation since it casts uints to f64
+    #
+    # There may be NaN in the similarity array if the both the cardinality
+    # and the vector are just zeros, in which case the intersection is 0 -> 0 / 0
+    #
+    # In these cases the fps are equal so the similarity *should be 1*, so we
+    # clamp the denominator, which is A | B (zero only if A & B is zero too).
+    return intersection / np.maximum(cardinalities + _popcount(y) - intersection, 1)
+
+
+def jt_isim_unpacked(arr: NDArray[np.integer]) -> float:
+    # cast is slower
+    return jt_isim_from_sum(
+        np.sum(arr, axis=0, dtype=np.uint64), len(arr)  # type: ignore
+    )
+
+
+def jt_isim_packed(fps: NDArray[np.integer], n_features: int | None = None) -> float:
+    # cast is slower
+    return jt_isim_from_sum(
+        np.sum(
+            unpack_fingerprints(fps, n_features),  # type: ignore
+            axis=0,
+            dtype=np.uint64,
+        ),
+        len(fps),
+    )
+
+
+def jt_isim_from_sum(linear_sum: NDArray[np.integer], n_objects: int) -> float:
+    r"""iSIM Tanimoto, from sum of rows of a fingerprint array and number of rows
+
+    iSIM Tanimoto was first propsed in:
+    https://pubs.rsc.org/en/content/articlelanding/2024/dd/d4dd00041b
+
+    :math:`iSIM_{JT}(X)` is an excellent :math:`O(N)` approximation of the average
+    Tanimoto similarity of a set of fingerprints.
+
+    Also equivalent to the complement of the Tanimoto diameter
+    :math:`iSIM_{JT}(X) = 1 - D_{JT}(X)`.
+
+    Parameters
+    ----------
+    c_total : np.ndarray
+              Sum of the elements from an array of fingerprints X, column-wise
+              c_total = np.sum(X, axis=0)
+
+    n_objects : int
+                Number of elements
+                n_objects = X.shape[0]
+
+    Returns
+    ----------
+    isim : float
+           iSIM Jaccard-Tanimoto value
+    """
+    if n_objects < 2:
+        warnings.warn(
+            f"Invalid n_objects = {n_objects} in isim. Expected n_objects >= 2",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+        return np.nan
+
+    x = linear_sum.astype(np.uint64, copy=False)
+    sum_kq = np.sum(x)
+    # isim of fingerprints that are all zeros should be 1 (they are all equal)
+    if sum_kq == 0:
+        return 1
+    sum_kqsq = np.dot(x, x)  # *dot* conserves dtype
+    a = (sum_kqsq - sum_kq) / 2  # 'a' is scalar f64
+    return a / (a + n_objects * sum_kq - sum_kqsq)