minhashlib 0.2.0__tar.gz → 0.2.1__tar.gz

{minhashlib-0.2.0 → minhashlib-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: minhashlib
-Version: 0.2.0
+Version: 0.2.1
 Summary: A fast and minimal minhashing based similarity checking library.
 Project-URL: Repository, https://github.com/ssavutu/minhashlib
 Project-URL: Bug Tracker, https://github.com/ssavutu/minhashlib/issues

{minhashlib-0.2.0 → minhashlib-0.2.1}/minhashlib/lsh.py

@@ -1,4 +1,5 @@
 from collections import defaultdict
+from functools import lru_cache
 
 import numpy as np
 from xxhash import xxh3_64_intdigest
@@ -28,10 +29,16 @@ def _false_negative_probability(threshold: float, bands: int, rows: int) -> floa
     return _integrate(prob, threshold, 1.0)
 
 
+@lru_cache(maxsize=None)
 def _optimal_bands_rows(
     threshold: float, num_perm: int, fp_weight: float = 0.5, fn_weight: float = 0.5
 ) -> tuple[int, int]:
-    """Pick ``(bands, rows)`` minimising weighted false-positive/false-negative error."""
+    """Pick ``(bands, rows)`` minimising weighted false-positive/false-negative error.
+
+    The search is a pure function of its arguments and costs ~tens of ms, so it is
+    memoised: constructing many LSH indexes with the same threshold/num_perm
+    (common in batch pipelines) recomputes nothing after the first call.
+    """
     best = (num_perm, 1)
     min_error = float("inf")
     for bands in range(1, num_perm + 1):

{minhashlib-0.2.0 → minhashlib-0.2.1}/minhashlib/minhash.py

@@ -57,18 +57,30 @@ class MinHash:
         n = len(document)
         if n == 0:
             return np.array([], dtype=np.int64)
-        if n < self.k:
+        p, k = self.p, self.k
+        hasher = xxh3_64_intdigest
+        if n < k:
             # Hash the whole string as a single shingle so documents shorter than
             # k remain distinguishable instead of collapsing to one signature.
-            return np.array([xxh3_64_intdigest(document.encode("utf-8")) % self.p], dtype=np.int64)
-
-        shingles = np.fromiter(
-            (
-                xxh3_64_intdigest(document[i : i + self.k].encode("utf-8")) % self.p
-                for i in range(n - self.k + 1)
-            ),
-            dtype=np.int64,
-        )
+            return np.array([hasher(document.encode("utf-8")) % p], dtype=np.int64)
+
+        count = n - k + 1
+        if document.isascii():
+            # ASCII: one byte per char, so encode once and slice the buffer instead
+            # of re-encoding each overlapping window. Byte slices are identical to
+            # ``document[i:i+k].encode()`` here, so signatures are unchanged.
+            data = document.encode("ascii")
+            shingles = np.fromiter(
+                (hasher(data[i : i + k]) % p for i in range(count)),
+                dtype=np.int64,
+                count=count,
+            )
+        else:
+            shingles = np.fromiter(
+                (hasher(document[i : i + k].encode("utf-8")) % p for i in range(count)),
+                dtype=np.int64,
+                count=count,
+            )
         return np.unique(shingles)
 
     def _generate_hash_parameters(self) -> NDArray[np.int64]:

{minhashlib-0.2.0 → minhashlib-0.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "minhashlib"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
   "numpy",
   "numba",