datasketch 1.9.0__tar.gz → 2.0.0__tar.gz

{datasketch-1.9.0 → datasketch-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: datasketch
-Version: 1.9.0
+Version: 2.0.0
 Summary: Probabilistic data structures for processing and searching very large datasets
 Project-URL: Homepage, https://ekzhu.github.io/datasketch
 Project-URL: Bug Tracker, https://github.com/ekzhu/datasketch/issues
@@ -23,10 +23,16 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
 Requires-Python: >=3.9
 Requires-Dist: numpy>=1.11
 Requires-Dist: scipy>=1.0.0
+Provides-Extra: aio
+Requires-Dist: aiounittest; extra == 'aio'
+Requires-Dist: motor>3.6.0; extra == 'aio'
 Provides-Extra: benchmark
+Requires-Dist: fonttools>=4.60.2; extra == 'benchmark'
 Requires-Dist: matplotlib>=3.1.2; extra == 'benchmark'
-Requires-Dist: nltk>=3.4.5; extra == 'benchmark'
+Requires-Dist: nltk>=3.4.5; (python_version < '3.10') and extra == 'benchmark'
+Requires-Dist: nltk>=3.9.4; (python_version >= '3.10') and extra == 'benchmark'
 Requires-Dist: pandas>=0.25.3; extra == 'benchmark'
+Requires-Dist: pillow>=12.2.0; (python_version >= '3.10') and extra == 'benchmark'
 Requires-Dist: pyfarmhash>=0.2.2; extra == 'benchmark'
 Requires-Dist: pyhash>=0.9.3; extra == 'benchmark'
 Requires-Dist: scikit-learn>=0.21.3; extra == 'benchmark'
@@ -48,11 +54,13 @@ Requires-Dist: mock>=2.0.0; extra == 'test'
 Requires-Dist: mockredispy; extra == 'test'
 Requires-Dist: nose-exclude>=0.5.0; extra == 'test'
 Requires-Dist: nose>=1.3.7; extra == 'test'
+Requires-Dist: pygments>=2.20.0; extra == 'test'
 Requires-Dist: pymongo>=3.9.0; extra == 'test'
-Requires-Dist: pytest; extra == 'test'
 Requires-Dist: pytest-asyncio; extra == 'test'
 Requires-Dist: pytest-cov; extra == 'test'
 Requires-Dist: pytest-rerunfailures; extra == 'test'
+Requires-Dist: pytest; (python_version < '3.10') and extra == 'test'
+Requires-Dist: pytest>=9.0.3; (python_version >= '3.10') and extra == 'test'
 Requires-Dist: redis>=2.10.0; extra == 'test'
 Description-Content-Type: text/x-rst
 
@@ -72,6 +80,17 @@ datasketch gives you probabilistic data structures that can process and
 search very large amount of data super fast, with little loss of
 accuracy.
 
+.. note::
+    **Version 2.0.0** changes the default MinHash permutation scheme to
+    ``"affine32"``, which fixes a similarity over-estimation bias on large
+    sets (`issue #212 <https://github.com/ekzhu/datasketch/issues/212>`__),
+    halves sketch memory, and speeds up updates by roughly 4x. A 64-bit
+    ``"affine64"`` scheme is available for billion-scale sets. Hash values
+    differ from earlier versions: rebuild persisted sketches and LSH
+    indexes, or pass ``MinHash(..., scheme="legacy")`` to interoperate with
+    existing data. See the `MinHash documentation
+    <https://ekzhu.github.io/datasketch/minhash.html>`__ for details.
+
 This package contains the following data sketches:
 
 +-------------------------+-----------------------------------------------+

{datasketch-1.9.0 → datasketch-2.0.0}/README.rst

@@ -14,6 +14,17 @@ datasketch gives you probabilistic data structures that can process and
 search very large amount of data super fast, with little loss of
 accuracy.
 
+.. note::
+    **Version 2.0.0** changes the default MinHash permutation scheme to
+    ``"affine32"``, which fixes a similarity over-estimation bias on large
+    sets (`issue #212 <https://github.com/ekzhu/datasketch/issues/212>`__),
+    halves sketch memory, and speeds up updates by roughly 4x. A 64-bit
+    ``"affine64"`` scheme is available for billion-scale sets. Hash values
+    differ from earlier versions: rebuild persisted sketches and LSH
+    indexes, or pass ``MinHash(..., scheme="legacy")`` to interoperate with
+    existing data. See the `MinHash documentation
+    <https://ekzhu.github.io/datasketch/minhash.html>`__ for details.
+
 This package contains the following data sketches:
 
 +-------------------------+-----------------------------------------------+

{datasketch-1.9.0 → datasketch-2.0.0}/datasketch/__init__.py

@@ -7,8 +7,9 @@ except importlib.metadata.PackageNotFoundError:
     _version = "0.0.0"  # Fallback for development mode
 __version__: Final[str] = _version
 
+from datasketch.aio import AsyncMinHashLSH  # Instantiation requires motor/redis.asyncio; import itself is always safe.
 from datasketch.b_bit_minhash import bBitMinHash
-from datasketch.hashfunc import sha1_hash32
+from datasketch.hashfunc import sha1_hash32, sha1_hash64
 from datasketch.hnsw import HNSW
 from datasketch.hyperloglog import HyperLogLog, HyperLogLogPlusPlus
 from datasketch.lean_minhash import LeanMinHash
@@ -23,9 +24,9 @@ from datasketch.weighted_minhash import WeightedMinHash, WeightedMinHashGenerato
 WeightedMinHashLSH = MinHashLSH
 WeightedMinHashLSHForest = MinHashLSHForest
 
-
 __all__ = [
     "HNSW",
+    "AsyncMinHashLSH",
     "HyperLogLog",
     "HyperLogLogPlusPlus",
     "LeanMinHash",
@@ -40,4 +41,5 @@ __all__ = [
     "WeightedMinHashLSHForest",
     "bBitMinHash",
     "sha1_hash32",
+    "sha1_hash64",
 ]

datasketch-2.0.0/datasketch/aio/__init__.py

@@ -0,0 +1,44 @@
+"""Async MinHash LSH module.
+
+This module provides asynchronous implementations of MinHash LSH for use with
+async storage backends like MongoDB (via motor) and Redis (via redis.asyncio).
+
+Example:
+    .. code-block:: python
+
+        import asyncio
+
+        from datasketch.aio import AsyncMinHashLSH
+        from datasketch import MinHash
+
+
+        async def main():
+            # prepickle=True lets you use non-bytes keys (e.g. str). With the
+            # default prepickle=False, keys passed to insert() must be bytes.
+            async with AsyncMinHashLSH(
+                storage_config={"type": "aiomongo", "mongo": {"host": "localhost", "port": 27017}},
+                threshold=0.5,
+                num_perm=128,
+                prepickle=True,
+            ) as lsh:
+                m = MinHash(num_perm=128)
+                m.update(b"data")
+                await lsh.insert("key", m)
+                result = await lsh.query(m)
+
+
+        asyncio.run(main())
+
+"""
+
+from datasketch.aio.lsh import (
+    AsyncMinHashLSH,
+    AsyncMinHashLSHDeleteSession,
+    AsyncMinHashLSHInsertionSession,
+)
+
+__all__ = [
+    "AsyncMinHashLSH",
+    "AsyncMinHashLSHDeleteSession",
+    "AsyncMinHashLSHInsertionSession",
+]

{datasketch-1.9.0/datasketch/experimental → datasketch-2.0.0/datasketch}/aio/lsh.py

@@ -1,13 +1,20 @@
+"""Asynchronous MinHash LSH implementation.
+
+This module provides AsyncMinHashLSH for use with async storage backends
+like MongoDB (via motor) and Redis (via redis.asyncio).
+"""
+
 import asyncio
 import pickle
 from itertools import chain
 from typing import Optional
 
-from datasketch.experimental.aio.storage import (
+from datasketch.aio.storage import (
     async_ordered_storage,
     async_unordered_storage,
 )
 from datasketch.lsh import _optimal_param
+from datasketch.minhash import _check_scheme_consistency
 from datasketch.storage import _random_name, unordered_storage
 
 
@@ -34,8 +41,6 @@ class AsyncMinHashLSH:
         MONGO = {"type": "aiomongo", "basename": "base_name_1", "mongo": {"host": "localhost", "port": 27017}}
 
     .. note::
-        * The module supports Python version >=3.6, and is currently experimental.
-          So the interface may change slightly in the future.
         * For main functionality of LSH algorithm see :class:`datasketch.MinHashLSH`.
         * For additional information see :ref:`minhash_lsh_at_scale` and :ref:`minhash_lsh_async`
     """
@@ -84,6 +89,10 @@ class AsyncMinHashLSH:
         self.hashranges = [(i * self.r, (i + 1) * self.r) for i in range(self.b)]
         self.hashtables = None
         self.keys = None
+        # The permutation scheme of the indexed MinHash, learned from the
+        # first insert. Note that an index attached to pre-existing external
+        # storage re-learns the scheme on its first insert.
+        self._minhash_scheme: Optional[str] = None
 
         self._lock = asyncio.Lock()
         self._initialized = False
@@ -129,7 +138,7 @@ class AsyncMinHashLSH:
         if self.keys is not None:
             self.keys.batch_size = value
         else:
-            raise AttributeError("AsyncMinHash is not initialized.")
+            raise AttributeError("AsyncMinHashLSH is not initialized.")
 
         for t in self.hashtables:
             t.batch_size = value
@@ -163,12 +172,6 @@ class AsyncMinHashLSH:
         if self.keys is None:
             await self._create_storages()
 
-        if not self.keys.initialized:
-            await self.keys
-
-        fs = (ht for ht in self.hashtables if not ht.initialized)
-        await asyncio.gather(*fs)
-
     async def close(self):
         """Cleanup client resources and disconnect from AsyncMinHashLSH storage."""
         async with self._lock:
@@ -189,41 +192,26 @@ class AsyncMinHashLSH:
 
         :param int batch_size: the size of chunks to use in insert_session mode (default=10000).
 
-        :return: datasketch.experimental.aio.lsh.AsyncMinHashLSHSession
+        :return: datasketch.aio.lsh.AsyncMinHashLSHInsertionSession
 
         Example:
             .. code-block:: python
 
-                from datasketch.experimental.aio.lsh import AsyncMinHashLSH
+                import asyncio
+                from datasketch.aio import AsyncMinHashLSH
                 from datasketch import MinHash
 
-
-                def chunk(it, size):
-                    it = iter(it)
-                    return iter(lambda: tuple(islice(it, size)), ())
-
-
-                _chunked_str = chunk((random.choice(string.ascii_lowercase) for _ in range(10000)), 4)
-                seq = frozenset(
-                    chain(
-                        ("".join(s) for s in _chunked_str),
-                        ("aahhb", "aahh", "aahhc", "aac", "kld", "bhg", "kkd", "yow", "ppi", "eer"),
-                    )
-                )
-                objs = [MinHash(16) for _ in range(len(seq))]
-                for e, obj in zip(seq, objs):
-                    for i in e:
-                        obj.update(i.encode("utf-8"))
-                data = [(e, m) for e, m in zip(seq, objs)]
-
-                _storage_config_redis = {"type": "aiomongo", "mongo": {"host": "localhost", "port": 27017}}
-
-
-                async def func():
-                    async with AsyncMinHashLSH(storage_config=_storage_config_redis, threshold=0.5, num_perm=16) as lsh:
+                async def main():
+                    storage_config = {"type": "aiomongo", "mongo": {"host": "localhost", "port": 27017}}
+                    async with AsyncMinHashLSH(
+                        storage_config=storage_config, threshold=0.5, num_perm=16, prepickle=True
+                    ) as lsh:
                         async with lsh.insertion_session(batch_size=1000) as session:
-                            fs = (session.insert(key, minhash, check_duplication=True) for key, minhash in data)
-                            await asyncio.gather(*fs)
+                            m = MinHash(num_perm=16)
+                            m.update(b"data")
+                            await session.insert("key", m)
+
+                asyncio.run(main())
 
         """
         return AsyncMinHashLSHInsertionSession(self, batch_size=batch_size)
@@ -232,47 +220,32 @@ class AsyncMinHashLSH:
         """Create a asynchronous context manager for fast removal of keys
         from index.
 
-        :param int batch_size: the size of chunks to use in insert_session mode (default=10000).
+        :param int batch_size: the size of chunks to use in delete_session mode (default=10000).
 
-        :return: datasketch.experimental.aio.lsh.AsyncMinHashLSHSession
+        :return: datasketch.aio.lsh.AsyncMinHashLSHDeleteSession
 
         Example:
             .. code-block:: python
 
-                from datasketch.experimental.aio.lsh import AsyncMinHashLSH
+                import asyncio
+                from datasketch.aio import AsyncMinHashLSH
                 from datasketch import MinHash
 
+                async def main():
+                    storage_config = {"type": "aiomongo", "mongo": {"host": "localhost", "port": 27017}}
+                    async with AsyncMinHashLSH(
+                        storage_config=storage_config, threshold=0.5, num_perm=16, prepickle=True
+                    ) as lsh:
+                        # Insert some data first
+                        m = MinHash(num_perm=16)
+                        m.update(b"data")
+                        await lsh.insert("key1", m)
 
-                def chunk(it, size):
-                    it = iter(it)
-                    return iter(lambda: tuple(islice(it, size)), ())
+                        # Delete using session
+                        async with lsh.delete_session(batch_size=100) as session:
+                            await session.remove("key1")
 
-
-                _chunked_str = chunk((random.choice(string.ascii_lowercase) for _ in range(10000)), 4)
-                seq = frozenset(
-                    chain(
-                        ("".join(s) for s in _chunked_str),
-                        ("aahhb", "aahh", "aahhc", "aac", "kld", "bhg", "kkd", "yow", "ppi", "eer"),
-                    )
-                )
-                objs = [MinHash(16) for _ in range(len(seq))]
-                for e, obj in zip(seq, objs):
-                    for i in e:
-                        obj.update(i.encode("utf-8"))
-                data = [(e, m) for e, m in zip(seq, objs)]
-
-                _storage_config_redis = {"type": "aiomongo", "mongo": {"host": "localhost", "port": 27017}}
-
-
-                async def func():
-                    async with AsyncMinHashLSH(storage_config=_storage_config_redis, threshold=0.5, num_perm=16) as lsh:
-                        async with lsh.insertion_session(batch_size=1000) as session:
-                            fs = (session.insert(key, minhash, check_duplication=True) for key, minhash in data)
-                            await asyncio.gather(*fs)
-
-                        async with lsh.delete_session(batch_size=3) as session:
-                            fs = (session.remove(key) for key in keys_to_remove)
-                            await asyncio.gather(*fs)
+                asyncio.run(main())
 
         """
         return AsyncMinHashLSHDeleteSession(self, batch_size=batch_size)
@@ -280,6 +253,7 @@ class AsyncMinHashLSH:
     async def _insert(self, key, minhash, check_duplication=True, buffer=False):
         if len(minhash) != self.h:
             raise ValueError("Expecting minhash with length %d, got %d" % (self.h, len(minhash)))
+        self._minhash_scheme = _check_scheme_consistency(getattr(self, "_minhash_scheme", None), minhash)
         if self._require_bytes_keys and not isinstance(key, bytes):
             raise TypeError(
                 f"prepickle=False requires bytes keys for non-dict storage, got {type(key).__name__}. "
@@ -288,7 +262,9 @@ class AsyncMinHashLSH:
         if self.prepickle:
             key = pickle.dumps(key)
 
-        if check_duplication and await self.has_key(key):
+        # `key` is already pickled at this point under prepickle=True; call the
+        # storage primitive directly so we don't re-pickle through has_key().
+        if check_duplication and await self.keys.has_key(key):
             raise ValueError("The given key already exists")
         Hs = [self._H(minhash.hashvalues[start:end]) for start, end in self.hashranges]
 
@@ -302,6 +278,7 @@ class AsyncMinHashLSH:
         """See :class:`datasketch.MinHashLSH`."""
         if len(minhash) != self.h:
             raise ValueError("Expecting minhash with length %d, got %d" % (self.h, len(minhash)))
+        _check_scheme_consistency(getattr(self, "_minhash_scheme", None), minhash)
 
         fs = (
             hashtable.get(self._H(minhash.hashvalues[start:end]))
@@ -314,6 +291,8 @@ class AsyncMinHashLSH:
 
     async def has_key(self, key):
         """See :class:`datasketch.MinHashLSH`."""
+        if self.prepickle:
+            key = pickle.dumps(key)
         return await self.keys.has_key(key)
 
     async def remove(self, key):
@@ -321,7 +300,12 @@ class AsyncMinHashLSH:
         await self._remove(key, buffer=False)
 
     async def _remove(self, key, buffer=False):
-        if not await self.has_key(key):
+        if self.prepickle:
+            key = pickle.dumps(key)
+
+        # `key` is already pickled here; call storage primitives directly so
+        # the existence check, lookup, and deletes all use the stored form.
+        if not await self.keys.has_key(key):
             raise ValueError("The given key does not exist")
 
         for H, hashtable in zip(await self.keys.get(key), self.hashtables):
@@ -345,6 +329,7 @@ class AsyncMinHashLSH:
     async def _query_b(self, minhash, b):
         if len(minhash) != self.h:
             raise ValueError("Expecting minhash with length %d, got %d" % (self.h, len(minhash)))
+        _check_scheme_consistency(getattr(self, "_minhash_scheme", None), minhash)
         if b > len(self.hashtables):
             raise ValueError("b must be less or equal to the number of hash tables")
         fs = []
@@ -352,7 +337,10 @@ class AsyncMinHashLSH:
             H = self._H(minhash.hashvalues[start:end])
             if await hashtable.has_key(H):
                 fs.append(hashtable.get(H))
-        return set(chain.from_iterable(await asyncio.gather(*fs)))  # candidates
+        candidates = set(chain.from_iterable(await asyncio.gather(*fs)))
+        if self.prepickle:
+            return {pickle.loads(key) for key in candidates}
+        return candidates
 
     async def get_counts(self):
         """See :class:`datasketch.MinHashLSH`."""
@@ -361,6 +349,10 @@ class AsyncMinHashLSH:
 
     async def get_subset_counts(self, *keys):
         """See :class:`datasketch.MinHashLSH`."""
+        # Keys in storage are pickled when prepickle is enabled, so we have to
+        # pickle the query keys to match the stored representation.
+        if self.prepickle:
+            keys = tuple(pickle.dumps(key) for key in keys)
         key_set = list(set(keys))
         hashtables = [unordered_storage({"type": "dict"}) for _ in range(self.b)]
         Hss = await self.keys.getmany(*key_set)

{datasketch-1.9.0/datasketch/experimental → datasketch-2.0.0/datasketch}/aio/storage.py

@@ -1,9 +1,23 @@
+"""Async storage backends for MinHash LSH.
+
+This module provides async storage implementations for use with AsyncMinHashLSH:
+- AsyncMongoListStorage / AsyncMongoSetStorage: MongoDB storage via motor
+- AsyncRedisListStorage / AsyncRedisSetStorage: Redis storage via redis.asyncio
+"""
+
 import asyncio
 import os
 from abc import ABCMeta
 from itertools import chain
 
-from datasketch.storage import OrderedStorage, RedisStorage, Storage, UnorderedStorage, _random_name
+from datasketch.storage import OrderedStorage, Storage, UnorderedStorage, _random_name
+
+# RedisStorage is only available when redis package is installed (optional dependency)
+# Import it conditionally to avoid ImportError when redis is not installed
+try:
+    from datasketch.storage import RedisStorage
+except ImportError:
+    RedisStorage = None
 
 ABC = ABCMeta("ABC", (object,), {})
 
@@ -24,6 +38,12 @@ except ImportError:
     redis = None
 
 
+__all__ = [
+    "async_ordered_storage",
+    "async_unordered_storage",
+]
+
+
 async def async_ordered_storage(config, name=None):
     tp = config["type"]
     if tp == "aiomongo":
@@ -55,9 +75,9 @@ if motor is not None and ReturnDocument is not None:
     class AsyncMongoBuffer:
         def __init__(self, aio_mongo_collection, batch_size):
             self._batch_size = batch_size
-            self._insert_documents_stack = tuple()
-            self._delete_by_key_documents_stack = tuple()
-            self._delete_by_val_documents_stack = tuple()
+            self._insert_documents_stack = []
+            self._delete_by_key_documents_stack = []
+            self._delete_by_val_documents_stack = []
             self._mongo_coll = aio_mongo_collection
 
         @property
@@ -73,28 +93,28 @@ if motor is not None and ReturnDocument is not None:
             if command == "insert":
                 if len(self._insert_documents_stack) >= self.batch_size:
                     await self.execute(command)
-                self._insert_documents_stack += (kwargs["obj"],)
+                self._insert_documents_stack.append(kwargs["obj"])
             elif command == "delete_by_key":
                 if len(self._delete_by_key_documents_stack) >= self.batch_size:
                     await self.execute(command)
-                self._delete_by_key_documents_stack += (kwargs["key"],)
+                self._delete_by_key_documents_stack.append(kwargs["key"])
             elif command == "delete_by_val":
                 if len(self._delete_by_val_documents_stack) >= self.batch_size:
                     await self.execute(command)
-                self._delete_by_val_documents_stack += (kwargs["val"],)
+                self._delete_by_val_documents_stack.append(kwargs["val"])
 
         async def execute(self, command):
             if command == "insert" and self._insert_documents_stack:
                 buffer = self._insert_documents_stack
-                self._insert_documents_stack = tuple()
+                self._insert_documents_stack = []
                 await self._mongo_coll.insert_many(buffer, ordered=False)
             elif command == "delete_by_key" and self._delete_by_key_documents_stack:
                 buffer = self._delete_by_key_documents_stack
-                self._delete_by_key_documents_stack = tuple()
+                self._delete_by_key_documents_stack = []
                 await self._mongo_coll.delete_many({"key": {"$in": buffer}})
             elif command == "delete_by_val" and self._delete_by_val_documents_stack:
                 buffer = self._delete_by_val_documents_stack
-                self._delete_by_val_documents_stack = tuple()
+                self._delete_by_val_documents_stack = []
                 await self._mongo_coll.delete_many({"vals": {"$in": buffer}})
 
         async def insert_one(self, **kwargs):
@@ -254,6 +274,9 @@ if motor is not None and ReturnDocument is not None:
         async def has_key(self, key):
             return bool(await self._collection.find_one({"key": key}))
 
+        async def getmany(self, *keys):
+            return await asyncio.gather(*(self.get(key) for key in keys))
+
         async def status(self):
             status = self._parse_config(self.config["mongo"])
             status.update({"keyspace_size": await self.size()})
@@ -285,7 +308,9 @@ if motor is not None and ReturnDocument is not None:
                 await self._collection.find_one_and_delete({"key": key, "vals": val})
 
 
-if redis is not None:
+# Redis-based async storage classes are only defined when both redis package
+# and RedisStorage are available (optional dependencies)
+if redis is not None and RedisStorage is not None:
 
     class AsyncRedisBuffer(redis.client.Pipeline):
         def __init__(self, connection_pool, response_callbacks, transaction, buffer_size, shard_hint=None):
@@ -304,7 +329,7 @@ if redis is not None:
 
         async def execute_command(self, *args, **kwargs):
             if len(self.command_stack) >= self._buffer_size:
-                self.execute()
+                await self.execute()
             await super(AsyncRedisBuffer, self).execute_command(*args, **kwargs)
 
     class AsyncRedisStorage(RedisStorage):
@@ -347,9 +372,8 @@ if redis is not None:
 
         async def getmany(self, *keys):
             pipe = self._redis.pipeline()
-            pipe.multi()
             for key in keys:
-                await self._get_items(pipe, self.redis_key(key))
+                pipe.lrange(self.redis_key(key), 0, -1)
             return await pipe.execute()
 
         @staticmethod
@@ -422,6 +446,12 @@ if redis is not None:
         async def _get_items(r, k):
             return await r.smembers(k)
 
+        async def getmany(self, *keys):
+            pipe = self._redis.pipeline()
+            for key in keys:
+                pipe.smembers(self.redis_key(key))
+            return await pipe.execute()
+
         async def remove_val(self, key, val, **kwargs):
             buffer = kwargs.pop("buffer", False)
             redis_key = self.redis_key(key)