datasketch 1.8.0__tar.gz → 1.9.0__tar.gz

{datasketch-1.8.0 → datasketch-1.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasketch
-Version: 1.8.0
+Version: 1.9.0
 Summary: Probabilistic data structures for processing and searching very large datasets
 Project-URL: Homepage, https://ekzhu.github.io/datasketch
 Project-URL: Bug Tracker, https://github.com/ekzhu/datasketch/issues
@@ -17,6 +17,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Database
 Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.9
@@ -49,6 +50,8 @@ Requires-Dist: nose-exclude>=0.5.0; extra == 'test'
 Requires-Dist: nose>=1.3.7; extra == 'test'
 Requires-Dist: pymongo>=3.9.0; extra == 'test'
 Requires-Dist: pytest; extra == 'test'
+Requires-Dist: pytest-asyncio; extra == 'test'
+Requires-Dist: pytest-cov; extra == 'test'
 Requires-Dist: pytest-rerunfailures; extra == 'test'
 Requires-Dist: redis>=2.10.0; extra == 'test'
 Description-Content-Type: text/x-rst
@@ -62,6 +65,9 @@ datasketch: Big Data Looks Small
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.598238.svg
    :target: https://zenodo.org/doi/10.5281/zenodo.598238
 
+.. image:: https://codecov.io/gh/ekzhu/datasketch/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/ekzhu/datasketch
+
 datasketch gives you probabilistic data structures that can process and
 search very large amount of data super fast, with little loss of
 accuracy.

{datasketch-1.8.0 → datasketch-1.9.0}/README.rst

@@ -7,6 +7,9 @@ datasketch: Big Data Looks Small
 .. image:: https://zenodo.org/badge/DOI/10.5281/zenodo.598238.svg
    :target: https://zenodo.org/doi/10.5281/zenodo.598238
 
+.. image:: https://codecov.io/gh/ekzhu/datasketch/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/ekzhu/datasketch
+
 datasketch gives you probabilistic data structures that can process and
 search very large amount of data super fast, with little loss of
 accuracy.

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/experimental/aio/lsh.py

@@ -60,6 +60,7 @@ class AsyncMinHashLSH:
         self._weights = weights
         self._params = params
         self.prepickle = storage_config["type"] == "aioredis" if prepickle is None else prepickle
+        self._require_bytes_keys = not self.prepickle
 
         if self._threshold > 1.0 or self._threshold < 0.0:
             raise ValueError("threshold must be in [0.0, 1.0]")
@@ -115,7 +116,9 @@ class AsyncMinHashLSH:
     def __setstate__(self, state):
         state["_lock"] = asyncio.Lock()
         self.__dict__ = state
-        self.__init__(self._threshold, self._num_perm, self._weights, self._params, self._storage_config)
+        self.__init__(
+            self._threshold, self._num_perm, self._weights, self._params, self._storage_config, self.prepickle
+        )
 
     @property
     def batch_size(self):
@@ -277,6 +280,11 @@ class AsyncMinHashLSH:
     async def _insert(self, key, minhash, check_duplication=True, buffer=False):
         if len(minhash) != self.h:
             raise ValueError("Expecting minhash with length %d, got %d" % (self.h, len(minhash)))
+        if self._require_bytes_keys and not isinstance(key, bytes):
+            raise TypeError(
+                f"prepickle=False requires bytes keys for non-dict storage, got {type(key).__name__}. "
+                "Either pass bytes keys or use prepickle=True for automatic serialization."
+            )
         if self.prepickle:
             key = pickle.dumps(key)
 

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/experimental/aio/storage.py

@@ -323,16 +323,19 @@ if redis is not None:
             )
             self._initialized = True
 
+        async def close(self):
+            await self._redis.aclose()
+
         @property
         def initialized(self):
             return self._initialized
 
     class AsyncRedisListStorage(OrderedStorage, AsyncRedisStorage):
         async def keys(self):
-            return await self._redis.hkeys(self._name)
+            return await self._redis.hkeys(self._name)  # type: ignore
 
         async def redis_keys(self):
-            return await self._redis.hvals(self._name)
+            return await self._redis.hvals(self._name)  # type: ignore
 
         def status(self):
             status = self._parse_config(self.config["redis"])
@@ -353,15 +356,26 @@ if redis is not None:
         async def _get_items(r, k):
             return await r.lrange(k, 0, -1)
 
-        async def remove(self, *keys):
-            await self._redis.hdel(self._name, *keys)
-            await self._redis.delete(*[self.redis_key(key) for key in keys])
+        async def remove(self, *keys, **kwargs):
+            buffer = kwargs.pop("buffer", False)
+            if buffer:
+                await self._remove(self._buffer, *keys)
+            else:
+                await self._remove(self._redis, *keys)
+
+        async def _remove(self, r, *keys):
+            await r.hdel(self._name, *keys)
+            await r.delete(*[self.redis_key(key) for key in keys])
 
-        async def remove_val(self, key, val):
+        async def remove_val(self, key, val, **kwargs):
+            buffer = kwargs.pop("buffer", False)
             redis_key = self.redis_key(key)
-            await self._redis.lrem(redis_key, val)
-            if not await self._redis.exists(redis_key):
-                await self._redis.hdel(self._name, redis_key)
+            if buffer:
+                await self._buffer.lrem(redis_key, val)
+            else:
+                await self._redis.lrem(redis_key, val)
+                if not await self._redis.exists(redis_key):  # type: ignore
+                    await self._redis.hdel(self._name, redis_key)  # type: ignore
 
         async def insert(self, key, *vals, **kwargs):
             # Using buffer=True outside of an `insertion_session`
@@ -380,7 +394,7 @@ if redis is not None:
             await r.rpush(redis_key, *values)
 
         async def size(self):
-            return await self._redis.hlen(self._name)
+            return await self._redis.hlen(self._name)  # type: ignore
 
         async def itemcounts(self):
             pipe = self._redis.pipeline()
@@ -395,7 +409,7 @@ if redis is not None:
             return await r.llen(k)
 
         async def has_key(self, key):
-            return await self._redis.hexists(self._name, key)
+            return await self._redis.hexists(self._name, key)  # type: ignore
 
         async def empty_buffer(self):
             await self._buffer.execute()
@@ -408,11 +422,15 @@ if redis is not None:
         async def _get_items(r, k):
             return await r.smembers(k)
 
-        async def remove_val(self, key, val):
+        async def remove_val(self, key, val, **kwargs):
+            buffer = kwargs.pop("buffer", False)
             redis_key = self.redis_key(key)
-            await self._redis.srem(redis_key, val)
-            if not await self._redis.exists(redis_key):
-                await self._redis.hdel(self._name, redis_key)
+            if buffer:
+                await self._buffer.srem(redis_key, val)
+            else:
+                await self._redis.srem(redis_key, val)
+                if not await self._redis.exists(redis_key):  # type: ignore
+                    await self._redis.hdel(self._name, redis_key)  # type: ignore
 
         async def _insert(self, r, key, *values):
             redis_key = self.redis_key(key)

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/lsh.py

@@ -3,12 +3,18 @@ from __future__ import annotations
 import pickle
 import struct
 from collections.abc import Hashable
-from typing import Callable, Optional, Union
+from typing import Callable, List, Optional, Union
 
 from scipy.integrate import quad as integrate
 
 from datasketch.minhash import MinHash
-from datasketch.storage import _random_name, ordered_storage, unordered_storage
+from datasketch.storage import (
+    OrderedStorage,
+    UnorderedStorage,
+    _random_name,
+    ordered_storage,
+    unordered_storage,
+)
 from datasketch.weighted_minhash import WeightedMinHash
 
 
@@ -183,7 +189,7 @@ class MinHashLSH:
             self._H = self._byteswap
 
         basename = storage_config.get("basename", _random_name(11))
-        self.hashtables = [
+        self.hashtables: List[UnorderedStorage] = [
             unordered_storage(
                 storage_config,
                 name=b"".join([basename, b"_bucket_", struct.pack(">H", i)]),
@@ -191,7 +197,7 @@ class MinHashLSH:
             for i in range(self.b)
         ]
         self.hashranges = [(i * self.r, (i + 1) * self.r) for i in range(self.b)]
-        self.keys = ordered_storage(storage_config, name=b"".join([basename, b"_keys"]))
+        self.keys: OrderedStorage = ordered_storage(storage_config, name=b"".join([basename, b"_keys"]))
 
     @property
     def buffer_size(self) -> int:
@@ -347,7 +353,7 @@ class MinHashLSH:
         """
         return type(self) is type(other) and self.h == other.h and self.b == other.b and self.r == other.r
 
-    def _merge(self, other: MinHashLSH, check_overlap: bool = False, buffer: bool = False) -> MinHashLSH:
+    def _merge(self, other: MinHashLSH, check_overlap: bool = False, buffer: bool = False) -> None:
         if self.__equivalent(other):
             if check_overlap and set(self.keys).intersection(set(other.keys)):
                 raise ValueError("The keys are overlapping, duplicate key exists.")
@@ -524,6 +530,8 @@ class MinHashLSH:
         return bytes(hs.byteswap().data)
 
     def _hashed_byteswap(self, hs):
+        if self.hashfunc is None:
+            raise RuntimeError("Hash function not configured.")
         return self.hashfunc(bytes(hs.byteswap().data))
 
     def _query_b(self, minhash, b):

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/lsh_bloom.py

@@ -252,9 +252,9 @@ class MinHashLSHBloom:
             raise ValueError("threshold must be in [0.0, 1.0]")
         if num_perm < 2:
             raise ValueError("Too few permutation functions")
-        if n <= 0:
+        if n is None or n <= 0:
             raise ValueError("n for LSHBloom must be >= 0")
-        if fp >= 1.0 or fp <= 0.0:
+        if fp is None or fp >= 1.0 or fp <= 0.0:
             raise ValueError("fp must be in (0.0, 1.0)")
         if save_dir is None:
             warnings.warn(

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/lshensemble.py

@@ -221,7 +221,8 @@ class MinHashLSHEnsemble:
         entries.sort(key=lambda e: e[2])
         curr_part = 0
         for key, minhash, size in entries:
-            if size > self.uppers[curr_part]:
+            u = self.uppers[curr_part]
+            if size > u:
                 curr_part += 1
             for r in self.indexes[curr_part]:
                 self.indexes[curr_part][r].insert(key, minhash)

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/minhash.py

@@ -3,15 +3,18 @@ from __future__ import annotations
 import copy
 import warnings
 from collections.abc import Generator, Iterable
-from typing import Callable, Optional
+from typing import TYPE_CHECKING, Callable, Optional, Union
 
 try:
     from typing import Literal  # py3.8+; if older, you can fallback to typing_extensions
-except Exception:
+except ImportError:
     from typing_extensions import Literal
 
 import numpy as np
 
+if TYPE_CHECKING:
+    from numpy.typing import ArrayLike
+
 # GPU backend
 try:
     import cupy as cp
@@ -114,8 +117,8 @@ class MinHash:
         gpu_mode: Literal["disable", "detect", "always"] = "disable",
         hashfunc: Callable = sha1_hash32,
         hashobj: Optional[object] = None,  # Deprecated.
-        hashvalues: Optional[Iterable] = None,
-        permutations: Optional[tuple[Iterable, Iterable]] = None,
+        hashvalues: Optional[ArrayLike] = None,
+        permutations: Optional[Union[tuple[ArrayLike, ArrayLike], ArrayLike]] = None,
     ) -> None:
         if hashvalues is not None:
             num_perm = len(hashvalues)
@@ -180,7 +183,7 @@ class MinHash:
             dtype=np.uint64,
         ).T
 
-    def _parse_hashvalues(self, hashvalues):
+    def _parse_hashvalues(self, hashvalues) -> np.ndarray:
         return np.array(hashvalues, dtype=np.uint64)
 
     def update(self, b) -> None:

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/storage.py

@@ -26,7 +26,7 @@ except ImportError:
     c_concurrent = None
 
 
-def ordered_storage(config, name=None):
+def ordered_storage(config, name=None) -> "OrderedStorage":
     """Return ordered storage system based on the specified config.
 
     The canonical example of such a storage container is
@@ -62,10 +62,10 @@ def ordered_storage(config, name=None):
         return RedisListStorage(config, name=name)
     if tp == "cassandra":
         return CassandraListStorage(config, name=name)
-    return None
+    raise ValueError(f"Unknown storage type: {tp}")
 
 
-def unordered_storage(config, name=None):
+def unordered_storage(config, name=None) -> "UnorderedStorage":
     """Return an unordered storage system based on the specified config.
 
     The canonical example of such a storage container is
@@ -100,7 +100,7 @@ def unordered_storage(config, name=None):
         return RedisSetStorage(config, name=name)
     if tp == "cassandra":
         return CassandraSetStorage(config, name=name)
-    return None
+    raise ValueError(f"Unknown storage type: {tp}")
 
 
 class Storage(ABC):
@@ -144,7 +144,7 @@ class Storage(ABC):
         pass
 
     @abstractmethod
-    def remove(self, *keys):
+    def remove(self, *keys, **kwargs):
         """Remove `keys` from storage."""
         pass
 
@@ -154,12 +154,12 @@ class Storage(ABC):
         pass
 
     @abstractmethod
-    def size(self):
+    def size(self) -> int:
         """Return size of storage with respect to number of keys."""
         pass
 
     @abstractmethod
-    def itemcounts(self, **kwargs):
+    def itemcounts(self, **kwargs) -> dict:
         """Returns the number of items stored under each key."""
         pass
 
@@ -168,6 +168,14 @@ class Storage(ABC):
         """Determines whether the key is in the storage or not."""
         pass
 
+    @property
+    def buffer_size(self) -> int:
+        return getattr(self, "_buffer_size", 50000)
+
+    @buffer_size.setter
+    def buffer_size(self, value: int):
+        self._buffer_size = value
+
     def status(self):
         return {"keyspace_size": len(self)}
 

{datasketch-1.8.0 → datasketch-1.9.0}/datasketch/weighted_minhash.py

@@ -133,14 +133,15 @@ class WeightedMinHashGenerator:
             WeightedMinHash: The weighted MinHash.
 
         """
-        if not isinstance(v, collections.abc.Iterable):
-            raise TypeError("Input vector must be an iterable")
+        if not isinstance(v, collections.abc.Sized):
+            raise TypeError("Input vector must be sized")
         if not len(v) == self.dim:
             raise ValueError("Input dimension mismatch, expecting %d" % self.dim)
         if not isinstance(v, np.ndarray):
             v = np.array(v, dtype=np.float32)
         elif v.dtype != np.float32:
             v = v.astype(np.float32)
+        v: np.ndarray = v
         hashvalues = np.zeros((self.sample_size, 2), dtype=int)
         vzeros = v == 0
         if vzeros.all():
@@ -226,9 +227,8 @@ class WeightedMinHashGenerator:
                 doc_argmin = np.argmin(doc_ln_a, axis=1)
                 doc_k = doc_cidx[doc_argmin]
 
-                all_hashvalues[it_doc] = np.zeros((self.sample_size, 2), dtype=int)
-
-                hashvalues = all_hashvalues[it_doc]
+                hashvalues = np.zeros((self.sample_size, 2), dtype=int)
+                all_hashvalues[it_doc] = hashvalues
                 hashvalues[:, 0], hashvalues[:, 1] = (
                     doc_k,
                     t[np.arange(self.sample_size), doc_begin + doc_argmin],

{datasketch-1.8.0 → datasketch-1.9.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "datasketch"
-version = "1.8.0"
+version = "1.9.0"
 description = "Probabilistic data structures for processing and searching very large datasets"
 readme = "README.rst"
 requires-python = ">=3.9"
@@ -21,6 +21,7 @@ classifiers = [
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
 ]
 dependencies = ["numpy>=1.11", "scipy>=1.0.0"]
 
@@ -44,11 +45,13 @@ test = [
   "mock>=2.0.0",
   "mockredispy",
   "coverage",
+  "pytest-cov",
   "pymongo>=3.9.0",
   "nose>=1.3.7",
   "nose-exclude>=0.5.0",
   "pytest",
   "pytest-rerunfailures",
+  "pytest-asyncio",
 ]
 experimental_aio = ["aiounittest", "motor>3.6.0"]
 
@@ -91,7 +94,6 @@ exclude = [
   "dist",
   "docs",
   "examples",
-  "travis",
   "datasketch/hyperloglog_const.py",
 ]
 
@@ -158,5 +160,38 @@ include = ["pyproject.toml", "README.rst", "LICENSE", "datasketch/**"]
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = ["--strict-markers", "--color=yes"]
+addopts = ["--strict-markers", "--color=yes", "--cov-report=xml"]
 testpaths = ["test"]
+asyncio_mode = "auto"
+
+[tool.pyright]
+include = ["datasketch"]
+exclude = [
+  "benchmark",
+  "docs",
+  "examples",
+  "test",
+  "travis",
+  "**/.venv/**",
+  "**/__pycache__",
+]
+pythonVersion = "3.9"
+typeCheckingMode = "basic" # todo: change to "strict" in future
+
+reportMissingImports = "none"
+reportUnusedVariable = "warning"
+reportAttributeAccessIssue = "none"
+reportOptionalMemberAccess = "none"
+reportGeneralTypeIssues = "none"
+reportArgumentType = "none"
+reportOptionalIterable = "none"
+reportReturnType = "none"
+reportRedeclaration = "none"
+reportOperatorIssue = "none"
+reportAssignmentType = "none"
+reportOptionalSubscript = "none"
+reportCallIssue = "none"
+
+[tool.coverage.run]
+source = ["datasketch"]
+omit = ["*/experimental/*", "*/tests/*", "*/test/*"]