minhashlib 0.1.0__tar.gz

minhashlib-0.1.0/.gitignore

@@ -0,0 +1,11 @@
+__pycache__/
+*.py[cod]
+*.nbi
+*.nbc
+
+.cache/
+.vscode/
+
+data/
+benchmark_outputs/
+benchmark_outputs_fast/

minhashlib-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sachin Avutu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

minhashlib-0.1.0/PKG-INFO

@@ -0,0 +1,114 @@
+Metadata-Version: 2.4
+Name: minhashlib
+Version: 0.1.0
+Summary: A fast and minimal minhashing based similarity checking library.
+Project-URL: Repository, https://github.com/ssavutu/minhashlib
+Project-URL: Bug Tracker, https://github.com/ssavutu/minhashlib/issues
+Author-email: Sachin Avutu <ssavutu@gmail.com>
+Maintainer-email: Sachin Avutu <ssavutu@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: deduplication,difference,entity resolution,minhash,minhashing,similarity
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python
+Requires-Python: >=3.10
+Requires-Dist: numba
+Requires-Dist: numpy
+Requires-Dist: xxhash
+Description-Content-Type: text/markdown
+
+# Minhashlib
+
+This is a minimal implementation of MinHashing as described in Jeffrey Ullman's book *Mining Massive Datasets*.
+
+## Current Benchmark Claim
+
+Based on the benchmark outputs in this repository:
+
+- On recent multi-seed CPU synthetic runs (seeds `42-46`), `minhashlib` builds signatures about `~6x` faster than `datasketch`.
+- Accuracy is comparable in magnitude (similar MAE scale and matching threshold-based metrics in these runs), though `datasketch` is slightly better on MAE in most synthetic scenarios.
+- Memory results are workload-dependent, so this project does not claim universal memory superiority.
+
+In short: this implementation is minimal and fast, with accuracy that is broadly comparable to `datasketch`, but outcomes vary depending on dataset and configuration.
+
+## Benchmark Suite
+
+Use `benchmarks/benchmark_claims_suite.py` to run comprehensive, reproducible benchmarks:
+
+- Multiple datasets (`synthetic`, `20newsgroups`, `wikipedia`, `ag_news`, `local`)
+- Multiple seeds with mean/std/95% CI
+- Metrics: MAE(Mean Average Error), Precision/Recall/F1 at threshold, Precision@K/Recall@K
+- Speed: build/pair-eval/retrieval latency and throughput
+- Memory: peak allocation and bytes/signature
+- Optional scaling sweeps over docs/number of hashes/doc length
+
+Example (full):
+
+```bash
+python3 benchmarks/benchmark_claims_suite.py
+  --datasets synthetic,20newsgroups,wikipedia
+  --wiki-dump-path data/simplewiki-latest-pages-articles.xml.bz2
+  --seeds 42,43,44
+  --p-values 2147483647,3037000493
+  --max-docs 2000
+  --random-pairs 3000
+  --num-queries 200
+  --include-scaling
+```
+
+Example (offline/local corpus only):
+
+```bash
+python3 benchmarks/benchmark_claims_suite.py
+  --datasets synthetic,local
+  --local-docs /path/to/docs.jsonl
+  --seeds 42,43,44
+```
+
+Outputs are written to `benchmark_outputs/` by default:
+
+- `raw_runs.json` / `raw_runs.csv`
+- `summary_stats.json` / `summary_stats.csv`
+- `run_metadata.json`
+- `skipped_runs.json`
+
+### Benchmark data setup
+
+Pull required benchmark datasets into local project paths:
+
+```bash
+python3 scripts/setup_benchmark_data.py
+```
+
+This prepares:
+
+- `data/simplewiki-latest-pages-articles.xml.bz2` (for Wikipedia benchmarks)
+- `.cache/scikit_learn_data` (for `20newsgroups` benchmarks)
+
+Optional flags:
+
+```bash
+python3 scripts/setup_benchmark_data.py --force
+python3 scripts/setup_benchmark_data.py --skip-wikipedia
+python3 scripts/setup_benchmark_data.py --skip-20newsgroups
+```
+
+### Individual Benchmarks
+
+You can run individual benchmarks instead of the full suite:
+
+```bash
+# Accuracy-only
+python3 benchmarks/benchmark_claims_accuracy.py --datasets synthetic,20newsgroups,wikipedia
+
+# Performance-only
+python3 benchmarks/benchmark_claims_performance.py --datasets synthetic,20newsgroups,wikipedia
+
+# Memory-only
+python3 benchmarks/benchmark_claims_memory.py --datasets synthetic,20newsgroups,wikipedia
+
+# Scaling-only (synthetic sweeps)
+python3 benchmarks/benchmark_claims_scaling.py --datasets synthetic
+```
+
+Each individual benchmark writes outputs under `benchmark_outputs/<test_name>/`.

minhashlib-0.1.0/README.md

@@ -0,0 +1,95 @@
+# Minhashlib
+
+This is a minimal implementation of MinHashing as described in Jeffrey Ullman's book *Mining Massive Datasets*.
+
+## Current Benchmark Claim
+
+Based on the benchmark outputs in this repository:
+
+- On recent multi-seed CPU synthetic runs (seeds `42-46`), `minhashlib` builds signatures about `~6x` faster than `datasketch`.
+- Accuracy is comparable in magnitude (similar MAE scale and matching threshold-based metrics in these runs), though `datasketch` is slightly better on MAE in most synthetic scenarios.
+- Memory results are workload-dependent, so this project does not claim universal memory superiority.
+
+In short: this implementation is minimal and fast, with accuracy that is broadly comparable to `datasketch`, but outcomes vary depending on dataset and configuration.
+
+## Benchmark Suite
+
+Use `benchmarks/benchmark_claims_suite.py` to run comprehensive, reproducible benchmarks:
+
+- Multiple datasets (`synthetic`, `20newsgroups`, `wikipedia`, `ag_news`, `local`)
+- Multiple seeds with mean/std/95% CI
+- Metrics: MAE(Mean Average Error), Precision/Recall/F1 at threshold, Precision@K/Recall@K
+- Speed: build/pair-eval/retrieval latency and throughput
+- Memory: peak allocation and bytes/signature
+- Optional scaling sweeps over docs/number of hashes/doc length
+
+Example (full):
+
+```bash
+python3 benchmarks/benchmark_claims_suite.py
+  --datasets synthetic,20newsgroups,wikipedia
+  --wiki-dump-path data/simplewiki-latest-pages-articles.xml.bz2
+  --seeds 42,43,44
+  --p-values 2147483647,3037000493
+  --max-docs 2000
+  --random-pairs 3000
+  --num-queries 200
+  --include-scaling
+```
+
+Example (offline/local corpus only):
+
+```bash
+python3 benchmarks/benchmark_claims_suite.py
+  --datasets synthetic,local
+  --local-docs /path/to/docs.jsonl
+  --seeds 42,43,44
+```
+
+Outputs are written to `benchmark_outputs/` by default:
+
+- `raw_runs.json` / `raw_runs.csv`
+- `summary_stats.json` / `summary_stats.csv`
+- `run_metadata.json`
+- `skipped_runs.json`
+
+### Benchmark data setup
+
+Pull required benchmark datasets into local project paths:
+
+```bash
+python3 scripts/setup_benchmark_data.py
+```
+
+This prepares:
+
+- `data/simplewiki-latest-pages-articles.xml.bz2` (for Wikipedia benchmarks)
+- `.cache/scikit_learn_data` (for `20newsgroups` benchmarks)
+
+Optional flags:
+
+```bash
+python3 scripts/setup_benchmark_data.py --force
+python3 scripts/setup_benchmark_data.py --skip-wikipedia
+python3 scripts/setup_benchmark_data.py --skip-20newsgroups
+```
+
+### Individual Benchmarks
+
+You can run individual benchmarks instead of the full suite:
+
+```bash
+# Accuracy-only
+python3 benchmarks/benchmark_claims_accuracy.py --datasets synthetic,20newsgroups,wikipedia
+
+# Performance-only
+python3 benchmarks/benchmark_claims_performance.py --datasets synthetic,20newsgroups,wikipedia
+
+# Memory-only
+python3 benchmarks/benchmark_claims_memory.py --datasets synthetic,20newsgroups,wikipedia
+
+# Scaling-only (synthetic sweeps)
+python3 benchmarks/benchmark_claims_scaling.py --datasets synthetic
+```
+
+Each individual benchmark writes outputs under `benchmark_outputs/<test_name>/`.

minhashlib-0.1.0/minhashlib/__init__.py

@@ -0,0 +1,3 @@
+from .diffchecker import DiffChecker
+
+__all__ = ["DiffChecker"]

minhashlib-0.1.0/minhashlib/diffchecker.py

@@ -0,0 +1,63 @@
+import numpy as np
+from numba import njit
+from xxhash import xxh3_64_intdigest
+from numpy.typing import NDArray
+
+
+@njit(cache=True)
+def _generate_signature_numba(
+    shingles: NDArray[np.int64], hash_params: NDArray[np.int64], p: int
+) -> NDArray[np.int64]:
+    num_perm = hash_params.shape[0]
+    sig = np.empty(num_perm, dtype=np.int64)
+    sig[:] = p
+
+    for i in range(shingles.shape[0]):
+        shingle = shingles[i]
+        for j in range(num_perm):
+            val = (hash_params[j, 0] * shingle + hash_params[j, 1]) % p
+            if val < sig[j]:
+                sig[j] = val
+
+    return sig
+
+
+class DiffChecker:
+    def __init__(self, p=3037000493, k=2, num_perm=128, seed: int | None = 42):
+        self.p = p
+        self.k = k
+        self.num_perm = num_perm
+        self.rng = np.random.default_rng(seed)
+        self.signatureMatrix = np.empty((num_perm, 0), dtype=np.int64)
+        self.hashParams = self.generateKHashParameters()
+
+    def generateKShingles(self, document: str) -> NDArray[np.int64]:
+        if len(document) < self.k:
+            return np.array([], dtype=np.int64)
+
+        shingles = np.fromiter(
+            (
+                xxh3_64_intdigest(document[i : i + self.k].encode("utf-8")) % self.p
+                for i in range(len(document) - self.k + 1)
+            ),
+            dtype=np.int64,
+        )
+
+        return np.unique(shingles)
+
+    def generateKHashParameters(self) -> NDArray[np.int64]:
+        a = self.rng.integers(1, self.p, size=self.num_perm, dtype=np.int64)
+        b = self.rng.integers(0, self.p, size=self.num_perm, dtype=np.int64)
+        return np.column_stack((a, b))
+
+    def generateSignature(self, document: str) -> NDArray[np.int64]:
+        shingles = self.generateKShingles(document)
+        return _generate_signature_numba(shingles, self.hashParams, self.p)
+
+    def appendDocumentAsSignature(self, document: str):
+        signature = self.generateSignature(document)
+        self.signatureMatrix = np.column_stack([self.signatureMatrix, signature])
+
+    @staticmethod
+    def checkJaccardSignatureSimilarity(a: NDArray[np.int64], b: NDArray[np.int64]) -> float:
+        return float(np.count_nonzero(a == b)) / float(a.size)

minhashlib-0.1.0/pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "minhashlib"
+version = "0.1.0"
+dependencies = [
+  "numpy",
+  "numba",
+  "xxhash",
+]
+requires-python = ">=3.10"
+authors = [
+  {name = "Sachin Avutu", email = "ssavutu@gmail.com"},
+]
+maintainers = [
+  {name = "Sachin Avutu", email = "ssavutu@gmail.com"}
+]
+description = "A fast and minimal minhashing based similarity checking library."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+keywords = ["minhashing", "minhash", "similarity", "difference", "entity resolution", "deduplication"]
+classifiers = [
+  "Development Status :: 3 - Alpha",
+  "Programming Language :: Python"
+]
+
+[project.urls]
+Repository = "https://github.com/ssavutu/minhashlib"
+"Bug Tracker" = "https://github.com/ssavutu/minhashlib/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["minhashlib"]
+
+[tool.hatch.build.targets.sdist]
+include = [
+  "minhashlib",
+  "README.md",
+  "LICENSE",
+  "pyproject.toml",
+]