lomboksimhash 0.1.0__tar.gz

lomboksimhash-0.1.0/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.4
+Name: lomboksimhash
+Version: 0.1.0
+Summary: Locality-sensitive hashing for near-duplicate document detection. SimHash, MinHash, LSH, SHA-256, and shingling.
+Author-email: CodingLombok <codinglombok@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/codinglombok/LombokSimHash
+Project-URL: Repository, https://github.com/codinglombok/LombokSimHash.git
+Project-URL: Changelog, https://github.com/codinglombok/LombokSimHash/blob/main/changelog.md
+Keywords: simhash,minhash,lsh,deduplication,hashing,locality-sensitive-hashing,near-duplicate,lombok
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.8; extra == "dev"
+Requires-Dist: mypy>=1.13; extra == "dev"
+
+# LombokSimHash (Python)
+
+Near-duplicate document detection for RAG pipelines — SimHash, MinHash, LSH, SHA-256, and shingling. Pure Python, zero dependencies.
+
+```bash
+pip install lomboksimhash
+```
+
+```python
+from lomboksimhash import simhash64_text, hamming64, MinHasher, jaccard_estimate, LshIndex
+
+a = simhash64_text("the quick brown fox jumps over the lazy dog")
+b = simhash64_text("the quick brown cat jumps over the lazy dog")
+print(hamming64(a, b))  # 8
+```
+
+Part of [LombokRAGFrameworks](https://github.com/codinglombok/LombokSimHash). Full docs in the [main repository](https://github.com/codinglombok/LombokSimHash).
+
+License: Apache-2.0

lomboksimhash-0.1.0/README.md

@@ -0,0 +1,19 @@
+# LombokSimHash (Python)
+
+Near-duplicate document detection for RAG pipelines — SimHash, MinHash, LSH, SHA-256, and shingling. Pure Python, zero dependencies.
+
+```bash
+pip install lomboksimhash
+```
+
+```python
+from lomboksimhash import simhash64_text, hamming64, MinHasher, jaccard_estimate, LshIndex
+
+a = simhash64_text("the quick brown fox jumps over the lazy dog")
+b = simhash64_text("the quick brown cat jumps over the lazy dog")
+print(hamming64(a, b))  # 8
+```
+
+Part of [LombokRAGFrameworks](https://github.com/codinglombok/LombokSimHash). Full docs in the [main repository](https://github.com/codinglombok/LombokSimHash).
+
+License: Apache-2.0

lomboksimhash-0.1.0/pyproject.toml

@@ -0,0 +1,87 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "lomboksimhash"
+version = "0.1.0"
+description = "Locality-sensitive hashing for near-duplicate document detection. SimHash, MinHash, LSH, SHA-256, and shingling."
+readme = "README.md"
+requires-python = ">=3.10"
+license = { text = "Apache-2.0" }
+authors = [
+    { name = "CodingLombok", email = "codinglombok@gmail.com" }
+]
+keywords = [
+    "simhash",
+    "minhash",
+    "lsh",
+    "deduplication",
+    "hashing",
+    "locality-sensitive-hashing",
+    "near-duplicate",
+    "lombok",
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Text Processing :: Indexing",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+]
+dependencies = []
+
+[project.urls]
+Homepage = "https://github.com/codinglombok/LombokSimHash"
+Repository = "https://github.com/codinglombok/LombokSimHash.git"
+Changelog = "https://github.com/codinglombok/LombokSimHash/blob/main/changelog.md"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.0",
+    "pytest-cov>=5.0",
+    "ruff>=0.8",
+    "mypy>=1.13",
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"
+
+[tool.coverage.run]
+branch = true
+source = ["lomboksimhash"]
+
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "N", "UP", "B", "A", "C4", "RET", "SIM", "PL"]
+ignore = ["PLR0913", "PLR2004"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["PLR0133", "PLR0912", "PLR0915", "SIM300"]
+
+[tool.mypy]
+strict = true
+files = ["src"]
+python_version = "3.10"
+warn_unreachable = true
+pretty = true

lomboksimhash-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

lomboksimhash-0.1.0/src/lomboksimhash/__init__.py

@@ -0,0 +1,48 @@
+"""LombokSimHash — Python Port.
+
+Near-duplicate document detection for RAG pipelines:
+SHA-256, FNV-1a 64, SimHash 64/128, MinHash, LSH, shingling.
+
+Zero dependencies. Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md;
+all ports are validated against vectors/lomboksimhash-vectors-v2.json.
+
+Part of LombokRAGFrameworks (Lombok Ecosystem).
+License: Apache-2.0
+"""
+
+from __future__ import annotations
+
+from ._core import FNV64_OFFSET, FNV64_PRIME, MASK64, WHITESPACE, fnv1a64, to_hex64, tokenize
+from ._lsh import LshIndex, band_hash
+from ._minhash import MinHasher, jaccard_estimate
+from ._sha256 import Sha256State, sha256, to_hex
+from ._shingle import Shingler, char_shingles, word_shingles
+from ._simhash import hamming64, hamming128, simhash64, simhash64_text, simhash128, simhash128_text
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "MASK64",
+    "FNV64_OFFSET",
+    "FNV64_PRIME",
+    "WHITESPACE",
+    "Sha256State",
+    "LshIndex",
+    "MinHasher",
+    "Shingler",
+    "band_hash",
+    "char_shingles",
+    "word_shingles",
+    "fnv1a64",
+    "hamming128",
+    "hamming64",
+    "jaccard_estimate",
+    "sha256",
+    "simhash128",
+    "simhash128_text",
+    "simhash64",
+    "simhash64_text",
+    "to_hex",
+    "to_hex64",
+    "tokenize",
+]

lomboksimhash-0.1.0/src/lomboksimhash/_core.py

@@ -0,0 +1,50 @@
+"""FNV-1a 64-bit and shared helpers (tokenization, hex formatting).
+
+Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
+"""
+
+from __future__ import annotations
+
+FNV64_OFFSET = 0xCBF29CE484222325
+FNV64_PRIME = 0x100000001B3
+MASK64 = (1 << 64) - 1
+
+# SPEC whitespace set = Unicode White_Space property (exact, cross-language).
+WHITESPACE = frozenset(
+    "\t\n\x0b\x0c\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005"
+    "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
+)
+
+
+def fnv1a64(data: bytes) -> int:
+    """FNV-1a 64-bit hash. offset 0xcbf29ce484222325, prime 0x100000001b3."""
+    h = FNV64_OFFSET
+    for b in data:
+        h ^= b
+        h = (h * FNV64_PRIME) & MASK64
+    return h
+
+
+def to_hex64(value: int) -> str:
+    """Format a u64 as 16 lowercase hex chars."""
+    return f"{value:016x}"
+
+
+def tokenize(text: str) -> list[str]:
+    """Split text on runs of the SPEC whitespace set; drop empty parts."""
+    return [t for t in _split_ws(text) if t]
+
+
+def _split_ws(text: str) -> list[str]:
+    parts: list[str] = []
+    cur: list[str] = []
+    for ch in text:
+        if ch in WHITESPACE:
+            if cur:
+                parts.append("".join(cur))
+                cur = []
+        else:
+            cur.append(ch)
+    if cur:
+        parts.append("".join(cur))
+    return parts

lomboksimhash-0.1.0/src/lomboksimhash/_lsh.py

@@ -0,0 +1,95 @@
+"""LSH — band+row locality-sensitive hashing over MinHash signatures.
+
+Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from ._core import FNV64_OFFSET, FNV64_PRIME, MASK64
+
+
+def _assert_config(bands: int, rows: int) -> None:
+    if bands < 1 or rows < 1:
+        raise ValueError("[ERR_LSH_CONFIG] bands and rows must be > 0")
+
+
+def _assert_sig_len(signature: Sequence[int], expected: int) -> None:
+    if len(signature) != expected:
+        raise ValueError(
+            f"[ERR_SIG_LEN] signature length must be bands * rows = {expected}"
+        )
+
+
+def band_hash(band_values: Sequence[int]) -> int:
+    """FNV-1a 64 over the little-endian bytes of each u64 in the band."""
+    h = FNV64_OFFSET
+    for val in band_values:
+        for byte_idx in range(8):
+            byte = (val >> (byte_idx * 8)) & 0xFF
+            h ^= byte
+            h = (h * FNV64_PRIME) & MASK64
+    return h
+
+
+class LshIndex:
+    """LSH index (band+row) over MinHash signatures.
+
+    query() returns de-duplicated doc IDs sorted ascending by code point.
+    """
+
+    __slots__ = ("_bands", "_rows", "_buckets")
+
+    def __init__(self, bands: int, rows: int) -> None:
+        _assert_config(bands, rows)
+        self._bands = bands
+        self._rows = rows
+        self._buckets: list[dict[int, list[str]]] = [{} for _ in range(bands)]
+
+    @property
+    def bands(self) -> int:
+        return self._bands
+
+    @property
+    def rows(self) -> int:
+        return self._rows
+
+    @property
+    def signature_len(self) -> int:
+        return self._bands * self._rows
+
+    def threshold(self) -> float:
+        """Approximate Jaccard threshold (1/bands)^(1/rows). Informational."""
+        base = 1.0 / self._bands
+        expo = 1.0 / self._rows
+        return float(base ** expo)
+
+    def insert(self, doc_id: str, signature: Sequence[int]) -> None:
+        """Insert a document signature into every band."""
+        _assert_sig_len(signature, self.signature_len)
+        for b in range(self._bands):
+            start = b * self._rows
+            key = band_hash(signature[start:start + self._rows])
+            self._buckets[b].setdefault(key, []).append(doc_id)
+
+    def query(self, signature: Sequence[int]) -> list[str]:
+        """Return sorted unique candidate doc IDs for the signature."""
+        _assert_sig_len(signature, self.signature_len)
+        candidates: set[str] = set()
+        for b in range(self._bands):
+            start = b * self._rows
+            key = band_hash(signature[start:start + self._rows])
+            candidates.update(self._buckets[b].get(key, ()))
+        return sorted(candidates)
+
+    def remove(self, doc_id: str, signature: Sequence[int]) -> None:
+        """Remove a doc ID from every band bucket touched by the signature."""
+        _assert_sig_len(signature, self.signature_len)
+        for b in range(self._bands):
+            start = b * self._rows
+            key = band_hash(signature[start:start + self._rows])
+            if key in self._buckets[b]:
+                self._buckets[b][key] = [
+                    d for d in self._buckets[b][key] if d != doc_id
+                ]

lomboksimhash-0.1.0/src/lomboksimhash/_minhash.py

@@ -0,0 +1,77 @@
+"""MinHash — Jaccard estimation via deterministic k-permutation family.
+
+Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from ._core import MASK64, fnv1a64
+
+MERSENNE_61 = (1 << 61) - 1
+LCG_SEED = 0x517CC1B727220A95
+LCG_MUL = 6364136223846793005
+LCG_INC = 1442695040888963407
+
+
+def _assert_num_perm(num_perm: int) -> None:
+    if num_perm < 1:
+        raise ValueError("[ERR_MINHASH_PERM] num_perm must be > 0")
+
+
+def _generate_hash_params(num_perm: int) -> list[tuple[int, int]]:
+    state = LCG_SEED
+    params: list[tuple[int, int]] = []
+    for _ in range(num_perm):
+        state = (state * LCG_MUL + LCG_INC) & MASK64
+        a = state % MERSENNE_61
+        if a == 0:  # pragma: no cover — defensive; LCG never yields a multiple of p
+            a = 1
+        state = (state * LCG_MUL + LCG_INC) & MASK64
+        b = state % MERSENNE_61
+        params.append((a, b))
+    return params
+
+
+def _universal_hash(a: int, b: int, x: int) -> int:
+    return (a * (x % MERSENNE_61) + b) % MERSENNE_61
+
+
+class MinHasher:
+    """MinHash signature generator with a deterministic permutation family."""
+
+    __slots__ = ("_num_perm", "_params", "signature")
+
+    def __init__(self, num_perm: int = 128) -> None:
+        _assert_num_perm(num_perm)
+        self._num_perm = num_perm
+        self._params = _generate_hash_params(num_perm)
+        self.signature = [MASK64] * num_perm
+
+    @property
+    def num_perm(self) -> int:
+        return self._num_perm
+
+    def add(self, token: str) -> None:
+        """Fold one token into the signature."""
+        h = fnv1a64(token.encode("utf-8"))
+        for i in range(self._num_perm):
+            a, b = self._params[i]
+            val = _universal_hash(a, b, h)
+            self.signature[i] = min(self.signature[i], val)
+
+    def add_tokens(self, tokens: Sequence[str]) -> None:
+        """Fold a sequence of tokens into the signature (order-independent)."""
+        for t in tokens:
+            self.add(t)
+
+
+def jaccard_estimate(sig_a: Sequence[int], sig_b: Sequence[int]) -> float:
+    """Estimated Jaccard = matched positions / signature length."""
+    if len(sig_a) != len(sig_b):
+        raise ValueError("[ERR_JACCARD_LEN] signatures must have same length")
+    if len(sig_a) == 0:
+        raise ValueError("[ERR_JACCARD_EMPTY] signatures must not be empty")
+    matches = sum(1 for a, b in zip(sig_a, sig_b, strict=False) if a == b)
+    return matches / len(sig_a)

lomboksimhash-0.1.0/src/lomboksimhash/_sha256.py

@@ -0,0 +1,134 @@
+"""SHA-256 (FIPS 180-4) — pure implementation.
+
+Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md; vectors in
+vectors/lomboksimhash-vectors-v2.json.
+"""
+
+from __future__ import annotations
+
+import struct
+
+_MASK32 = 0xFFFFFFFF
+
+_K_SHA = [
+    0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
+    0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
+    0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
+    0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
+    0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
+    0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
+    0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
+    0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
+    0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
+    0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
+    0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
+    0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
+    0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
+    0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
+    0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
+    0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
+]
+
+_H_INIT = [
+    0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
+    0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19,
+]
+
+
+def _rotr32(x: int, n: int) -> int:
+    return ((x >> n) | (x << (32 - n))) & _MASK32
+
+
+def _sha256_compress(state: list[int], block: bytes) -> None:
+    w = [0] * 64
+    for i in range(16):
+        w[i] = struct.unpack(">I", block[i * 4:i * 4 + 4])[0]
+    for i in range(16, 64):
+        s0 = _rotr32(w[i - 15], 7) ^ _rotr32(w[i - 15], 18) ^ (w[i - 15] >> 3)
+        s1 = _rotr32(w[i - 2], 17) ^ _rotr32(w[i - 2], 19) ^ (w[i - 2] >> 10)
+        w[i] = (w[i - 16] + s0 + w[i - 7] + s1) & _MASK32
+
+    a, b, c, d, e, f, g, h = state
+
+    for i in range(64):
+        s1 = _rotr32(e, 6) ^ _rotr32(e, 11) ^ _rotr32(e, 25)
+        ch = (e & f) ^ ((~e & _MASK32) & g)
+        temp1 = (h + s1 + ch + _K_SHA[i] + w[i]) & _MASK32
+        s0 = _rotr32(a, 2) ^ _rotr32(a, 13) ^ _rotr32(a, 22)
+        maj = (a & b) ^ (a & c) ^ (b & c)
+        temp2 = (s0 + maj) & _MASK32
+
+        h = g
+        g = f
+        f = e
+        e = (d + temp1) & _MASK32
+        d = c
+        c = b
+        b = a
+        a = (temp1 + temp2) & _MASK32
+
+    for i, v in enumerate((a, b, c, d, e, f, g, h)):
+        state[i] = (state[i] + v) & _MASK32
+
+
+class Sha256State:
+    """Incremental SHA-256 hasher (streaming)."""
+
+    __slots__ = ("_state", "_buffer", "_total_len")
+
+    def __init__(self) -> None:
+        self._state = list(_H_INIT)
+        self._buffer = bytearray()
+        self._total_len = 0
+
+    def update(self, data: bytes | bytearray | memoryview) -> None:
+        self._total_len += len(data)
+        self._buffer.extend(data)
+        while len(self._buffer) >= 64:
+            block = bytes(self._buffer[:64])
+            del self._buffer[:64]
+            _sha256_compress(self._state, block)
+
+    def digest(self) -> bytes:
+        """Finalize (without destroying state) and return the 32-byte digest."""
+        state = list(self._state)
+        buffer = bytearray(self._buffer)
+        bit_len = self._total_len * 8
+        buffer.append(0x80)
+        while len(buffer) % 64 != 56:
+            buffer.append(0x00)
+        buffer.extend(bit_len.to_bytes(8, "big"))
+        for i in range(0, len(buffer), 64):
+            _sha256_compress(state, bytes(buffer[i:i + 64]))
+        return b"".join(v.to_bytes(4, "big") for v in state)
+
+    def hexdigest(self) -> str:
+        return self.digest().hex()
+
+    def finalize(self) -> bytes:
+        """Finalize and return the 32-byte digest."""
+        return self.digest()
+
+
+def sha256(data: bytes | str) -> str:
+    """Compute SHA-256 of bytes or UTF-8 text. Returns lowercase hex string."""
+    if isinstance(data, str):
+        data = data.encode("utf-8")
+    state = list(_H_INIT)
+    bit_len = len(data) * 8
+
+    padded = bytearray(data)
+    padded.append(0x80)
+    while len(padded) % 64 != 56:
+        padded.append(0x00)
+    padded.extend(bit_len.to_bytes(8, "big"))
+
+    for i in range(0, len(padded), 64):
+        _sha256_compress(state, bytes(padded[i:i + 64]))
+
+    return "".join(f"{v:08x}" for v in state)
+
+
+def to_hex(digest: bytes) -> str:
+    """Convert digest bytes to lowercase hex string."""
+    return digest.hex()

lomboksimhash-0.1.0/src/lomboksimhash/_shingle.py

@@ -0,0 +1,76 @@
+"""Shingling — character/word n-gram generation (streaming).
+
+Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
+"""
+
+from __future__ import annotations
+
+from ._core import tokenize
+
+_SHINGLE_N_MSG = "[ERR_SHINGLE_N] shingle size must be > 0"
+
+
+def _assert_n(n: int) -> None:
+    if n < 1:
+        raise ValueError(_SHINGLE_N_MSG)
+
+
+class Shingler:
+    """Streaming shingle generator.
+
+    Character mode slices Unicode code points; word mode slices tokens
+    produced by the SPEC tokenizer and joins them with a single U+0020.
+    finish() returns unique shingles sorted by code point.
+    """
+
+    __slots__ = ("_mode", "_n", "_buffer")
+
+    def __init__(self, mode: str = "character", n: int = 3) -> None:
+        _assert_n(n)
+        if mode not in ("character", "word"):
+            raise ValueError("mode must be 'character' or 'word'")
+        self._mode = mode
+        self._n = n
+        self._buffer = ""
+
+    @property
+    def mode(self) -> str:
+        return self._mode
+
+    @property
+    def n(self) -> int:
+        return self._n
+
+    def feed(self, text: str) -> None:
+        self._buffer += text
+
+    def finish(self) -> list[str]:
+        """Return sorted unique shingles."""
+        if self._mode == "character":
+            chars = list(self._buffer)
+            count = len(chars)
+            return sorted(
+                {"".join(chars[i:i + self._n]) for i in range(count - self._n + 1)}
+            )
+        words = tokenize(self._buffer)
+        count = len(words)
+        return sorted(
+            {" ".join(words[i:i + self._n]) for i in range(count - self._n + 1)}
+        )
+
+    def feed_finish(self, text: str) -> list[str]:
+        """Convenience: feed(text) then finish()."""
+        self.feed(text)
+        return self.finish()
+
+
+def char_shingles(text: str, n: int) -> list[str]:
+    """Convenience: sorted unique character n-grams (code points)."""
+    _assert_n(n)
+    return Shingler("character", n).feed_finish(text)
+
+
+def word_shingles(text: str, n: int) -> list[str]:
+    """Convenience: sorted unique word n-grams."""
+    _assert_n(n)
+    return Shingler("word", n).feed_finish(text)

lomboksimhash-0.1.0/src/lomboksimhash/_simhash.py

@@ -0,0 +1,77 @@
+"""SimHash — 64/128-bit locality-sensitive fingerprinting via FNV-1a.
+
+Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from ._core import fnv1a64, tokenize
+
+
+def simhash64(tokens: Sequence[str]) -> int:
+    """64-bit SimHash over token list. Bit i set iff weight[i] > 0 (LSB first)."""
+    v = [0] * 64
+    for token in tokens:
+        h = fnv1a64(token.encode("utf-8"))
+        for i in range(64):
+            if (h >> i) & 1:
+                v[i] += 1
+            else:
+                v[i] -= 1
+    result = 0
+    for i in range(64):
+        if v[i] > 0:
+            result |= 1 << i
+    return result
+
+
+def simhash64_text(text: str) -> int:
+    """Tokenize by the SPEC whitespace set, then simhash64."""
+    return simhash64(tokenize(text))
+
+
+def simhash128(tokens: Sequence[str]) -> tuple[int, int]:
+    """128-bit SimHash. Returns (hi, lo).
+
+    lo = simhash of fnv1a64(utf8(token)); hi = simhash of
+    fnv1a64(utf8(token) || 0xFF).
+    """
+    v = [0] * 128
+    for token in tokens:
+        raw = token.encode("utf-8")
+        h1 = fnv1a64(raw)
+        h2 = fnv1a64(raw + b"\xff")
+        for i in range(64):
+            if (h1 >> i) & 1:
+                v[i] += 1
+            else:
+                v[i] -= 1
+            if (h2 >> i) & 1:
+                v[64 + i] += 1
+            else:
+                v[64 + i] -= 1
+    lo = 0
+    hi = 0
+    for i in range(64):
+        if v[i] > 0:
+            lo |= 1 << i
+        if v[64 + i] > 0:
+            hi |= 1 << i
+    return (hi, lo)
+
+
+def simhash128_text(text: str) -> tuple[int, int]:
+    """Tokenize by the SPEC whitespace set, then simhash128."""
+    return simhash128(tokenize(text))
+
+
+def hamming64(a: int, b: int) -> int:
+    """Hamming distance between two 64-bit values."""
+    return bin(a ^ b).count("1")
+
+
+def hamming128(a: tuple[int, int], b: tuple[int, int]) -> int:
+    """Hamming distance between two 128-bit (hi, lo) pairs."""
+    return bin(a[0] ^ b[0]).count("1") + bin(a[1] ^ b[1]).count("1")

lomboksimhash-0.1.0/src/lomboksimhash.egg-info/PKG-INFO

@@ -0,0 +1,47 @@
+Metadata-Version: 2.4
+Name: lomboksimhash
+Version: 0.1.0
+Summary: Locality-sensitive hashing for near-duplicate document detection. SimHash, MinHash, LSH, SHA-256, and shingling.
+Author-email: CodingLombok <codinglombok@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/codinglombok/LombokSimHash
+Project-URL: Repository, https://github.com/codinglombok/LombokSimHash.git
+Project-URL: Changelog, https://github.com/codinglombok/LombokSimHash/blob/main/changelog.md
+Keywords: simhash,minhash,lsh,deduplication,hashing,locality-sensitive-hashing,near-duplicate,lombok
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Scientific/Engineering :: Information Analysis
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"
+Requires-Dist: ruff>=0.8; extra == "dev"
+Requires-Dist: mypy>=1.13; extra == "dev"
+
+# LombokSimHash (Python)
+
+Near-duplicate document detection for RAG pipelines — SimHash, MinHash, LSH, SHA-256, and shingling. Pure Python, zero dependencies.
+
+```bash
+pip install lomboksimhash
+```
+
+```python
+from lomboksimhash import simhash64_text, hamming64, MinHasher, jaccard_estimate, LshIndex
+
+a = simhash64_text("the quick brown fox jumps over the lazy dog")
+b = simhash64_text("the quick brown cat jumps over the lazy dog")
+print(hamming64(a, b))  # 8
+```
+
+Part of [LombokRAGFrameworks](https://github.com/codinglombok/LombokSimHash). Full docs in the [main repository](https://github.com/codinglombok/LombokSimHash).
+
+License: Apache-2.0

lomboksimhash-0.1.0/src/lomboksimhash.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+pyproject.toml
+src/lomboksimhash/__init__.py
+src/lomboksimhash/_core.py
+src/lomboksimhash/_lsh.py
+src/lomboksimhash/_minhash.py
+src/lomboksimhash/_sha256.py
+src/lomboksimhash/_shingle.py
+src/lomboksimhash/_simhash.py
+src/lomboksimhash.egg-info/PKG-INFO
+src/lomboksimhash.egg-info/SOURCES.txt
+src/lomboksimhash.egg-info/dependency_links.txt
+src/lomboksimhash.egg-info/requires.txt
+src/lomboksimhash.egg-info/top_level.txt
+tests/test_api.py
+tests/test_vectors.py

lomboksimhash-0.1.0/src/lomboksimhash.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

lomboksimhash-0.1.0/src/lomboksimhash.egg-info/requires.txt

@@ -0,0 +1,6 @@
+
+[dev]
+pytest>=8.0
+pytest-cov>=5.0
+ruff>=0.8
+mypy>=1.13

lomboksimhash-0.1.0/src/lomboksimhash.egg-info/top_level.txt

@@ -0,0 +1 @@
+lomboksimhash

lomboksimhash-0.1.0/tests/test_api.py

@@ -0,0 +1,216 @@
+"""Unit + API tests for the Python port (error contract, edge cases)."""
+
+from __future__ import annotations
+
+import pytest
+
+from lomboksimhash import (
+    LshIndex,
+    MinHasher,
+    Shingler,
+    char_shingles,
+    fnv1a64,
+    hamming64,
+    hamming128,
+    jaccard_estimate,
+    sha256,
+    simhash64,
+    simhash64_text,
+    simhash128,
+    simhash128_text,
+    tokenize,
+    word_shingles,
+)
+from lomboksimhash._core import to_hex64
+from lomboksimhash._sha256 import Sha256State, to_hex
+from lomboksimhash._sha256 import sha256 as sha256_fn
+
+FOX = "the quick brown fox jumps over the lazy dog"
+
+
+class TestSha256:
+    def test_reference_digests(self) -> None:
+        assert sha256(b"") == (
+            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+        )
+        assert sha256("abc") == (
+            "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"
+        )
+
+    def test_str_and_bytes_identical(self) -> None:
+        assert sha256("héllo") == sha256("héllo".encode())
+
+
+class TestFnv:
+    def test_offset(self) -> None:
+        assert fnv1a64(b"") == 0xCBF29CE484222325
+
+    def test_known(self) -> None:
+        assert f"{fnv1a64(b'hello'):016x}" == "a430d84680aabd0b"
+
+
+class TestTokenize:
+    def test_spec_whitespace(self) -> None:
+        assert tokenize("a\u00a0b\u3000c\u000bd") == ["a", "b", "c", "d"]
+
+    def test_excludes_zwsp_and_bom(self) -> None:
+        assert tokenize("a\u200bb") == ["a\u200bb"]
+        assert tokenize("a\ufeffb") == ["a\ufeffb"]
+
+    def test_empty(self) -> None:
+        assert tokenize("") == []
+        assert tokenize(" \t\n") == []
+
+
+class TestShingling:
+    def test_char_and_word(self) -> None:
+        assert char_shingles("abcde", 3) == ["abc", "bcd", "cde"]
+        assert word_shingles("a   b\t\tc\n\nc", 2) == ["a b", "b c", "c c"]
+
+    def test_streaming_pure(self) -> None:
+        s = Shingler("character", 3)
+        s.feed("abc")
+        s.feed("de")
+        assert s.finish() == ["abc", "bcd", "cde"]
+        assert s.finish() == ["abc", "bcd", "cde"]  # finish() idempotent
+
+    def test_n_exceeds_len(self) -> None:
+        assert char_shingles("ab", 5) == []
+        assert word_shingles("a b", 4) == []
+
+    def test_err_shingle_n(self) -> None:
+        with pytest.raises(ValueError, match=r"^\[ERR_SHINGLE_N\] shingle size must be > 0$"):
+            Shingler("character", 0)
+        with pytest.raises(ValueError, match=r"^\[ERR_SHINGLE_N\]"):
+            char_shingles("abc", 0)
+        with pytest.raises(ValueError, match=r"^\[ERR_SHINGLE_N\]"):
+            word_shingles("abc", 0)
+
+    def test_bad_mode(self) -> None:
+        with pytest.raises(ValueError, match="mode"):
+            Shingler("bogus", 2)
+
+
+class TestSimhash:
+    def test_known(self) -> None:
+        assert f"{simhash64_text(FOX):016x}" == "cab7991c5475edee"
+        assert f"{simhash64_text(FOX.replace('fox', 'cat')):016x}" == "c2e7991c1465e56f"
+
+    def test_empty(self) -> None:
+        assert simhash64([]) == 0
+        assert simhash64_text("   \t\n") == 0
+        assert simhash128([]) == (0, 0)
+
+    def test_hamming(self) -> None:
+        fox = simhash64_text(FOX)
+        cat = simhash64_text(FOX.replace("fox", "cat"))
+        assert hamming64(fox, cat) == 8
+        assert hamming64(fox, fox) == 0
+
+    def test_128_lo_equals_64(self) -> None:
+        hi, lo = simhash128_text(FOX)
+        assert lo == simhash64_text(FOX)
+        assert hi >= 0 and hi < (1 << 64)
+        assert hamming128((hi, lo), (hi, lo)) == 0
+
+
+class TestMinHash:
+    def test_identical(self) -> None:
+        a = MinHasher(128)
+        b = MinHasher(128)
+        a.add_tokens(["the", "quick", "brown", "fox"])
+        b.add_tokens(["the", "quick", "brown", "fox"])
+        assert jaccard_estimate(a.signature, b.signature) == 1.0
+
+    def test_order_independent_and_idempotent(self) -> None:
+        a = MinHasher(64)
+        b = MinHasher(64)
+        a.add_tokens(["hello", "world"])
+        b.add_tokens(["world", "hello", "hello"])
+        assert a.signature == b.signature
+
+    def test_errors(self) -> None:
+        with pytest.raises(ValueError, match=r"^\[ERR_MINHASH_PERM\] num_perm must be > 0$"):
+            MinHasher(0)
+        a = MinHasher(8)
+        b = MinHasher(16)
+        with pytest.raises(ValueError, match=r"^\[ERR_JACCARD_LEN\]"):
+            jaccard_estimate(a.signature, b.signature)
+        with pytest.raises(ValueError, match=r"^\[ERR_JACCARD_EMPTY\]"):
+            jaccard_estimate([], [])
+
+
+class TestLsh:
+    def test_errors(self) -> None:
+        with pytest.raises(ValueError, match=r"^\[ERR_LSH_CONFIG\]"):
+            LshIndex(0, 5)
+        with pytest.raises(ValueError, match=r"^\[ERR_LSH_CONFIG\]"):
+            LshIndex(20, 0)
+        idx = LshIndex(5, 2)
+        expected = r"^\[ERR_SIG_LEN\] signature length must be bands \* rows = 10$"
+        with pytest.raises(ValueError, match=expected):
+            idx.insert("d", [1, 2, 3])
+        with pytest.raises(ValueError, match=r"^\[ERR_SIG_LEN\]"):
+            idx.query([1])
+
+    def test_sorted_query_and_remove(self) -> None:
+        idx = LshIndex(3, 1)
+        sig = [1, 2, 3]
+        for doc in ("b_doc", "a_doc", "c_doc", "a_doc"):
+            idx.insert(doc, sig)
+        assert idx.query(sig) == ["a_doc", "b_doc", "c_doc"]
+        idx.remove("a_doc", sig)
+        assert idx.query(sig) == ["b_doc", "c_doc"]
+
+    def test_threshold(self) -> None:
+        assert abs(LshIndex(20, 5).threshold() - 0.5493) < 0.01
+        assert LshIndex(1, 1).threshold() == 1.0
+
+
+class TestSha256State:
+    def test_streaming_equals_one_shot(self) -> None:
+        data = "Rust TypeScript Python Go PHP — satu kontrak bit-identical 🌍".encode()
+        for split in (0, 1, 31, 55, 56, 57, len(data)):
+            st = Sha256State()
+            st.update(data[:split])
+            st.update(data[split:])
+            assert st.hexdigest() == sha256_fn(data), f"split {split}"
+
+    def test_digest_non_destructive(self) -> None:
+        st = Sha256State()
+        st.update(b"hello")
+        first = st.digest()
+        assert st.digest() == first  # digest() does not destroy state
+        st.update(b" world")
+        assert st.hexdigest() == sha256_fn(b"hello world")
+
+    def test_finalize_and_to_hex(self) -> None:
+        st = Sha256State()
+        st.update(b"abc")
+        digest = st.finalize()
+        assert to_hex(digest) == sha256_fn(b"abc")
+        assert len(digest) == 32
+
+
+class TestAccessors:
+    def test_shingler_accessors(self) -> None:
+        s = Shingler("word", 4)
+        assert s.mode == "word"
+        assert s.n == 4
+
+    def test_lsh_accessors(self) -> None:
+        idx = LshIndex(7, 3)
+        assert idx.bands == 7
+        assert idx.rows == 3
+        assert idx.signature_len == 21
+
+    def test_to_hex64(self) -> None:
+        assert to_hex64(0) == "0" * 16
+        assert to_hex64(0xCBF29CE484222325) == "cbf29ce484222325"
+
+
+class TestMinHashAccessors:
+    def test_num_perm_accessor(self) -> None:
+        a = MinHasher(42)
+        assert a.num_perm == 42
+        assert len(a.signature) == 42

lomboksimhash-0.1.0/tests/test_vectors.py

@@ -0,0 +1,134 @@
+"""Vector tests — consume the ONE shared cross-language vectors file."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from lomboksimhash import (
+    LshIndex,
+    MinHasher,
+    band_hash,
+    char_shingles,
+    fnv1a64,
+    jaccard_estimate,
+    sha256,
+    simhash64_text,
+    simhash128_text,
+    word_shingles,
+)
+
+VECTORS = (
+    Path(__file__).resolve().parents[3] / "vectors" / "lomboksimhash-vectors-v2.json"
+)
+
+
+@pytest.fixture(scope="module")
+def doc() -> dict[str, Any]:
+    with VECTORS.open(encoding="utf-8") as f:
+        data = json.load(f)
+    assert data["schema"] == "lomboksimhash-vectors-v2"
+    return data
+
+
+def hex64(v: int) -> str:
+    return f"{v:016x}"
+
+
+def sig_of(hexes: list[str]) -> list[int]:
+    return [int(h, 16) for h in hexes]
+
+
+def test_sha256_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["sha256"]:
+        data = case.get("input")
+        raw = data.encode("utf-8") if isinstance(data, str) else bytes.fromhex(case["input_hex"])
+        assert sha256(raw) == case["expected_hex"], f"case {case['id']}"
+
+
+def test_fnv_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["fnv1a64"]:
+        got = hex64(fnv1a64(bytes.fromhex(case["input_hex"])))
+        assert got == case["expected_hex"], f"case {case['id']}"
+
+
+def test_shingle_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["shingle"]:
+        fn = char_shingles if case["mode"] == "character" else word_shingles
+        got = fn(case["input"], case["n"])
+        assert got == case["expected"], f"case {case['id']}"
+
+
+def test_simhash64_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["simhash64"]:
+        got = hex64(simhash64_text(case["input"]))
+        assert got == case["expected_hex"], f"case {case['id']}"
+
+
+def test_simhash128_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["simhash128"]:
+        hi, lo = simhash128_text(case["input"])
+        assert hex64(hi) == case["hi_hex"], f"case {case['id']} hi"
+        assert hex64(lo) == case["lo_hex"], f"case {case['id']} lo"
+        assert hex64(hi) + hex64(lo) == case["hex"], f"case {case['id']} hex"
+
+
+def test_minhash_signature_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["minhash_signature"]:
+        mh = MinHasher(case["num_perm"])
+        mh.add_tokens(case["tokens"])
+        got = [hex64(v) for v in mh.signature]
+        assert got == case["signature_hex"], f"case {case['id']}"
+
+
+def test_minhash_jaccard_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["minhash_jaccard"]:
+        num_perm = case["num_perm"]
+        a = MinHasher(num_perm)
+        b = MinHasher(num_perm)
+        a.add_tokens(case["set_a"])
+        b.add_tokens(case["set_b"])
+        matches = sum(1 for x, y in zip(a.signature, b.signature, strict=False) if x == y)
+        assert matches == case["expected_matches"], f"case {case['id']}"
+        assert jaccard_estimate(a.signature, b.signature) == matches / num_perm
+
+
+def test_lsh_bandhash_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["lsh_bandhash"]:
+        values = sig_of(case["band_values_hex"])
+        got = hex64(band_hash(values))
+        assert got == case["expected_key_hex"], f"case {case['id']}"
+        idx = LshIndex(1, len(values))
+        idx.insert("probe", values)
+        assert "probe" in idx.query(values)
+
+
+def test_lsh_index_vectors(doc: dict[str, Any]) -> None:
+    for case in doc["cases"]["lsh_index"]:
+        idx = LshIndex(case["bands"], case["rows"])
+        for ins in case["inserts"]:
+            idx.insert(ins["doc_id"], sig_of(ins["signature_hex"]))
+        got = idx.query(sig_of(case["query_hex"]))
+        assert got == case["expected"], f"case {case['id']}"
+        remove = case.get("remove")
+        if remove:
+            idx.remove(remove["doc_id"], sig_of(remove["signature_hex"]))
+            after = idx.query(sig_of(case["query_hex"]))
+            assert after == case["expected_after_remove"], f"case {case['id']}"
+
+
+def test_vector_file_deterministic(doc: dict[str, Any]) -> None:
+    # The committed file must equal a fresh regeneration (spec §7).
+    repo = Path(__file__).resolve().parents[3]
+    result = subprocess.run(
+        [sys.executable, str(repo / "vectors" / "generate_vectors.py"), "--check"],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr