pyhyperminhash 0.2.0__cp39-abi3-win_arm64.whl

pyhyperminhash/__init__.py

@@ -0,0 +1,5 @@
+from .pyhyperminhash import *
+
+__doc__ = pyhyperminhash.__doc__
+if hasattr(pyhyperminhash, "__all__"):
+    __all__ = pyhyperminhash.__all__

pyhyperminhash/__init__.pyi

@@ -0,0 +1,59 @@
+import typing
+from typing_extensions import final
+
+class _SupportsRead(typing.Protocol):
+    def read(self, n: int, /) -> bytes: ...
+
+__all__ = [
+    "__version_info__",
+    "Entry",
+    "Sketch",
+    "__version__",
+    "__profile__",
+    "__hyperminhash_version__",
+]
+
+@final
+class Entry:
+    def add(self, obj: typing.Hashable, /) -> None: ...
+    def add_bytes(self, obj: bytes, /) -> None: ...
+    def __iadd__(self, value: typing.Hashable, /) -> "Entry": ...
+    def add_reader(self, src: _SupportsRead, /) -> int: ...
+    def fork(self) -> "Entry": ...
+    def _digest(self) -> int: ...
+
+@final
+class Sketch:
+    @staticmethod
+    def load(buf: bytes, /) -> "Sketch": ...
+    @staticmethod
+    def from_iter(src: typing.Iterator[typing.Hashable], /) -> "Sketch": ...
+    def save(self) -> bytes: ...
+    def add(self, obj: typing.Hashable, /) -> None: ...
+    def add_reader(self, src: _SupportsRead, /) -> int: ...
+    def add_entry(self, entry: Entry, /) -> None: ...
+    def cardinality(self) -> float: ...
+    def union(self, other: "Sketch", /) -> None: ...
+    def intersection(self, other: "Sketch", /, *, fast: bool = False) -> float: ...
+    def similarity(self, other: "Sketch", /, *, fast: bool = False) -> float: ...
+    def intersection_many(
+        self, others: typing.Iterable["Sketch"], /
+    ) -> list[float]: ...
+    def similarity_many(self, others: typing.Iterable["Sketch"], /) -> list[float]: ...
+    def __bool__(self) -> bool: ...
+    def __int__(self) -> int: ...
+    def __len__(self) -> int: ...
+    def __float__(self) -> float: ...
+    def __iadd__(self, value: typing.Hashable, /) -> "Sketch": ...
+    def __ior__(self, value: "Sketch", /) -> "Sketch": ...
+    def __or__(self, value: "Sketch", /) -> "Sketch": ...
+    def __lt__(self, value: "Sketch", /) -> bool: ...
+    def __le__(self, value: "Sketch", /) -> bool: ...
+    def __gt__(self, value: "Sketch", /) -> bool: ...
+    def __ge__(self, value: "Sketch", /) -> bool: ...
+
+def __version_info__() -> str: ...
+
+__version__: str
+__hyperminhash_version__: str
+__profile__: str

pyhyperminhash-0.2.0.dist-info/METADATA

@@ -0,0 +1,188 @@
+Metadata-Version: 2.4
+Name: pyhyperminhash
+Version: 0.2.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: MIT License
+License-File: LICENSE
+Summary: Very fast, constant memory-footprint cardinality approximation, including intersection
+Author-email: Lukas Lueg <lukas.lueg@gmail.com>
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Homepage, https://github.com/lukaslueg/pyhyperminhash
+
+# Hyperminhash for Python
+
+[![Docs](https://docs.rs/hyperminhash/badge.svg)](https://docs.rs/hyperminhash)
+[![PyPI](https://badge.fury.io/py/pyhyperminhash.svg)](https://pypi.org/project/pyhyperminhash/)
+
+Very fast, constant memory-footprint cardinality approximation, including intersection
+and union operation.
+
+The class `Sketch` can be used to count unique elements that were encountered during
+the instance's lifetime. Unlike e.g. when using a `set`, the memory consumed by the
+`Sketch`-instance does _not_ grow as elements are added; each `Sketch`-instance
+consumes approximately 32kb of memory, independent of the number of elements.
+
+### Installation
+
+```bash
+pip install pyhyperminhash
+```
+
+### Basic usage
+
+```python
+# Construct an empty Sketch
+sk = pyhyperminhash.Sketch()
+assert not sk  # The Sketch is empty
+sk.add('Foo')  # Add elements
+sk.add('Bar')
+sk.add(42)
+assert sk  # The Sketch is not empty
+assert float(sk) == sk.cardinality()  # Approximately 3.0
+assert int(sk) == len(sk)  # Approximately 3
+```
+
+```python
+# Sketches can be combined...
+
+sk2 = pyhyperminhash.Sketch()
+sk2.add('Foobar')
+sk2 |= sk  # sk2 now holds all elements that were in `sk` or `sk2`
+
+sk3 = pyhyperminhash.Sketch()
+sk3.add(42)
+sk3.add('Foo')
+assert sk3.intersection(sk2) > 0.0  # Approximately 1.0, due to `Foo`
+
+# `fast=True` uses the faster upstream estimate with a slightly looser
+# correction model; for most inputs the difference is tiny.
+assert sk3.intersection(sk2, fast=True) > 0.0
+assert sk3.similarity(sk2, fast=True) > 0.0
+
+# Compare one Sketch against many others
+others = [sk, sk2, sk3]
+assert sk3.intersection_many(others) == [sk3.intersection(other) for other in others]
+assert sk3.similarity_many(others) == [sk3.similarity(other) for other in others]
+```
+
+```python
+# Sketches can be stored for later use
+
+buf = sk.save()  # Serialize the Sketch into a bytes-buffer
+new_sk = pyhyperminhash.Sketch.load(buf)  # Deserialize a Sketch
+assert len(new_sk) == len(sk)
+```
+
+```python
+# Sketches can be constructed from iterators directly, which avoids
+# materializing all objects simultaneously.
+
+# This is also faster than adding elements one by one.
+sk = pyhyperminhash.Sketch.from_iter(iter(range(100)))
+
+# Read all lines; uses binary mode to avoid Unicode encoding+decoding
+with open('complaints.txt', 'rb') as f:
+    sk = pyhyperminhash.Sketch.from_iter(f)
+
+# Add an entire file-like object
+with open('complaints.txt', 'rb') as f:
+    sk.add_reader(f)
+```
+
+```python
+# Entry-objects can be used to construct what represents a single
+# object from multiple parts. They consume very little memory.
+
+# Add sub-objects that don't fit into memory
+e = pyhyperminhash.Entry()
+with open('complaints.txt', 'rb') as f:
+    e.add(f.read(4096))  # Add two parts to this Entry
+    e.add(f.read(4096))
+    sk.add_entry(e)  # This counts as one object
+
+sk2 = pyhyperminhash.Sketch()
+sk2.add_entry(e)  # Entry-objects can be added to multiple Sketches
+
+# Entry-objects can be "forked" at the state they are currently at
+e2 = e.fork()
+e2.add("Foo")
+sk.add_entry(e2)  # This is a new object in `Sketch`...
+
+e3 = e.fork()
+e3.add("Bar")
+sk.add_entry(e3)  # So is this, without re-computing `e`
+```
+
+It is very much preferred to provide `bytes` to `Sketch.add()` and `Entry.add()`,
+in order to avoid double-hashing the content. Also, notice that the Python interpreter
+probably randomizes hash-values at interpreter-startup (in order to prevent certain kinds
+of DOS-attacks). The randomness introduced here means that the count-approximation provided
+by this package may not be stable from run to run.
+
+See the documentation for the underlying [implementation](https://docs.rs/hyperminhash) for additional information.
+
+### Usage as a sqlite3 aggregate function
+
+```python
+class PyHyperminhashCounter:
+    def __init__(self):
+        self.sk = pyhyperminhash.Sketch()
+
+    def step(self, *args) -> None:
+        self.sk += args
+
+    def finalize(self) -> int:
+        return int(self.sk)
+
+
+con = sqlite3.connect(":memory:")
+# Create the function for the current connection
+con.create_aggregate("pyhmh", -1, PyHyperminhashCounter)
+
+# Some dummy data
+cur = con.execute("CREATE TABLE test(i, j)")
+cur.executemany("INSERT INTO test(i, j) VALUES(?, ?)",
+                ((f'foo{i % 4291}bar', i % 819)
+                 for i in range(1000000)))
+
+# Returns exactly 502047
+cur.execute("SELECT COUNT(*) FROM (SELECT DISTINCT i, j FROM test)")
+print(cur.fetchone()[0])
+
+# Returns approximately 500000, but faster
+cur.execute("SELECT pyhmh(i, j) FROM test")
+print(cur.fetchone()[0])
+```
+
+### Performance examples
+
+Reading a file of 50 million lines, ~1,000,000 unique elements, 20 characters per line.
+
+Method                     | Wall-clock time | Memory consumed | Count      |
+---------------------------|-----------------|-----------------|------------|
+Count lines                | 2.3 seconds     | _nil_           | 50,000,000 |
+`set()`                    | 14.99 seconds   | ~144 megabytes  | 1,048,576  |
+`Sketch.from_iter()`       | 1.8 seconds     | ~32 kilobytes   | 1,041,936  |
+
+For repeatable local benchmarks of the Python-facing API, use the `pyperf` suite:
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+maturin develop -r
+pip install pyperf
+python3 examples/bench.py
+```
+
+### FAQ
+
+* Can I extract an element that was previously added from a `Sketch`?
+  * No.
+* Can I check if an element has previously been added to a `Sketch`?
+  * No.
+* Wow, why use this, then?
+  * It's very fast at counting unique things. And it does not use much memory.
+

pyhyperminhash-0.2.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+pyhyperminhash/__init__.py,sha256=bhBaICIKP_QxoP3VoISd10yFR3QQO5oFP5ChuNs3fgg,139
+pyhyperminhash/__init__.pyi,sha256=7sIsBMiXTgseywi-qCoydl5YwvZGX-PbK735OP9LYQQ,2129
+pyhyperminhash/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyhyperminhash/pyhyperminhash.pyd,sha256=00adBqDCjxv5SdQTbM0fg2rt-oL1_5SMsusNNTFZFn4,370688
+pyhyperminhash-0.2.0.dist-info/METADATA,sha256=keCqhejg2ciHcUi6kC9egyq8B1QCWJHmuzqMxQJGbmU,6338
+pyhyperminhash-0.2.0.dist-info/WHEEL,sha256=2muIeEoy5DUTfuM3RFMnd0bGlZCGVPOH8LjuzkFAm8s,95
+pyhyperminhash-0.2.0.dist-info/licenses/LICENSE,sha256=5F9XyCGLCpCvSZi0l7noUGu1ISKges2jCstiLZhstdU,1106
+pyhyperminhash-0.2.0.dist-info/sboms/pyhyperminhash.cyclonedx.json,sha256=qLvrRuQPbdG8qM3dqTAuxZVsA_rTAYIk55F_x_aerPc,71343
+pyhyperminhash-0.2.0.dist-info/RECORD,,

pyhyperminhash-0.2.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: maturin (1.13.1)
+Root-Is-Purelib: false
+Tag: cp39-abi3-win_arm64

pyhyperminhash-0.2.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Lukas Lueg (lukas.lueg@gmail.com)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.