lomboksimhash 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ """LombokSimHash — Python Port.
2
+
3
+ Near-duplicate document detection for RAG pipelines:
4
+ SHA-256, FNV-1a 64, SimHash 64/128, MinHash, LSH, shingling.
5
+
6
+ Zero dependencies. Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md;
7
+ all ports are validated against vectors/lomboksimhash-vectors-v2.json.
8
+
9
+ Part of LombokRAGFrameworks (Lombok Ecosystem).
10
+ License: Apache-2.0
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from ._core import FNV64_OFFSET, FNV64_PRIME, MASK64, WHITESPACE, fnv1a64, to_hex64, tokenize
16
+ from ._lsh import LshIndex, band_hash
17
+ from ._minhash import MinHasher, jaccard_estimate
18
+ from ._sha256 import Sha256State, sha256, to_hex
19
+ from ._shingle import Shingler, char_shingles, word_shingles
20
+ from ._simhash import hamming64, hamming128, simhash64, simhash64_text, simhash128, simhash128_text
21
+
22
+ __version__ = "0.1.0"
23
+
24
+ __all__ = [
25
+ "MASK64",
26
+ "FNV64_OFFSET",
27
+ "FNV64_PRIME",
28
+ "WHITESPACE",
29
+ "Sha256State",
30
+ "LshIndex",
31
+ "MinHasher",
32
+ "Shingler",
33
+ "band_hash",
34
+ "char_shingles",
35
+ "word_shingles",
36
+ "fnv1a64",
37
+ "hamming128",
38
+ "hamming64",
39
+ "jaccard_estimate",
40
+ "sha256",
41
+ "simhash128",
42
+ "simhash128_text",
43
+ "simhash64",
44
+ "simhash64_text",
45
+ "to_hex",
46
+ "to_hex64",
47
+ "tokenize",
48
+ ]
lomboksimhash/_core.py ADDED
@@ -0,0 +1,50 @@
1
+ """FNV-1a 64-bit and shared helpers (tokenization, hex formatting).
2
+
3
+ Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ FNV64_OFFSET = 0xCBF29CE484222325
9
+ FNV64_PRIME = 0x100000001B3
10
+ MASK64 = (1 << 64) - 1
11
+
12
+ # SPEC whitespace set = Unicode White_Space property (exact, cross-language).
13
+ WHITESPACE = frozenset(
14
+ "\t\n\x0b\x0c\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005"
15
+ "\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
16
+ )
17
+
18
+
19
+ def fnv1a64(data: bytes) -> int:
20
+ """FNV-1a 64-bit hash. offset 0xcbf29ce484222325, prime 0x100000001b3."""
21
+ h = FNV64_OFFSET
22
+ for b in data:
23
+ h ^= b
24
+ h = (h * FNV64_PRIME) & MASK64
25
+ return h
26
+
27
+
28
+ def to_hex64(value: int) -> str:
29
+ """Format a u64 as 16 lowercase hex chars."""
30
+ return f"{value:016x}"
31
+
32
+
33
+ def tokenize(text: str) -> list[str]:
34
+ """Split text on runs of the SPEC whitespace set; drop empty parts."""
35
+ return [t for t in _split_ws(text) if t]
36
+
37
+
38
+ def _split_ws(text: str) -> list[str]:
39
+ parts: list[str] = []
40
+ cur: list[str] = []
41
+ for ch in text:
42
+ if ch in WHITESPACE:
43
+ if cur:
44
+ parts.append("".join(cur))
45
+ cur = []
46
+ else:
47
+ cur.append(ch)
48
+ if cur:
49
+ parts.append("".join(cur))
50
+ return parts
lomboksimhash/_lsh.py ADDED
@@ -0,0 +1,95 @@
1
+ """LSH — band+row locality-sensitive hashing over MinHash signatures.
2
+
3
+ Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from collections.abc import Sequence
9
+
10
+ from ._core import FNV64_OFFSET, FNV64_PRIME, MASK64
11
+
12
+
13
+ def _assert_config(bands: int, rows: int) -> None:
14
+ if bands < 1 or rows < 1:
15
+ raise ValueError("[ERR_LSH_CONFIG] bands and rows must be > 0")
16
+
17
+
18
+ def _assert_sig_len(signature: Sequence[int], expected: int) -> None:
19
+ if len(signature) != expected:
20
+ raise ValueError(
21
+ f"[ERR_SIG_LEN] signature length must be bands * rows = {expected}"
22
+ )
23
+
24
+
25
+ def band_hash(band_values: Sequence[int]) -> int:
26
+ """FNV-1a 64 over the little-endian bytes of each u64 in the band."""
27
+ h = FNV64_OFFSET
28
+ for val in band_values:
29
+ for byte_idx in range(8):
30
+ byte = (val >> (byte_idx * 8)) & 0xFF
31
+ h ^= byte
32
+ h = (h * FNV64_PRIME) & MASK64
33
+ return h
34
+
35
+
36
+ class LshIndex:
37
+ """LSH index (band+row) over MinHash signatures.
38
+
39
+ query() returns de-duplicated doc IDs sorted ascending by code point.
40
+ """
41
+
42
+ __slots__ = ("_bands", "_rows", "_buckets")
43
+
44
+ def __init__(self, bands: int, rows: int) -> None:
45
+ _assert_config(bands, rows)
46
+ self._bands = bands
47
+ self._rows = rows
48
+ self._buckets: list[dict[int, list[str]]] = [{} for _ in range(bands)]
49
+
50
+ @property
51
+ def bands(self) -> int:
52
+ return self._bands
53
+
54
+ @property
55
+ def rows(self) -> int:
56
+ return self._rows
57
+
58
+ @property
59
+ def signature_len(self) -> int:
60
+ return self._bands * self._rows
61
+
62
+ def threshold(self) -> float:
63
+ """Approximate Jaccard threshold (1/bands)^(1/rows). Informational."""
64
+ base = 1.0 / self._bands
65
+ expo = 1.0 / self._rows
66
+ return float(base ** expo)
67
+
68
+ def insert(self, doc_id: str, signature: Sequence[int]) -> None:
69
+ """Insert a document signature into every band."""
70
+ _assert_sig_len(signature, self.signature_len)
71
+ for b in range(self._bands):
72
+ start = b * self._rows
73
+ key = band_hash(signature[start:start + self._rows])
74
+ self._buckets[b].setdefault(key, []).append(doc_id)
75
+
76
+ def query(self, signature: Sequence[int]) -> list[str]:
77
+ """Return sorted unique candidate doc IDs for the signature."""
78
+ _assert_sig_len(signature, self.signature_len)
79
+ candidates: set[str] = set()
80
+ for b in range(self._bands):
81
+ start = b * self._rows
82
+ key = band_hash(signature[start:start + self._rows])
83
+ candidates.update(self._buckets[b].get(key, ()))
84
+ return sorted(candidates)
85
+
86
+ def remove(self, doc_id: str, signature: Sequence[int]) -> None:
87
+ """Remove a doc ID from every band bucket touched by the signature."""
88
+ _assert_sig_len(signature, self.signature_len)
89
+ for b in range(self._bands):
90
+ start = b * self._rows
91
+ key = band_hash(signature[start:start + self._rows])
92
+ if key in self._buckets[b]:
93
+ self._buckets[b][key] = [
94
+ d for d in self._buckets[b][key] if d != doc_id
95
+ ]
@@ -0,0 +1,77 @@
1
+ """MinHash — Jaccard estimation via deterministic k-permutation family.
2
+
3
+ Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from collections.abc import Sequence
9
+
10
+ from ._core import MASK64, fnv1a64
11
+
12
+ MERSENNE_61 = (1 << 61) - 1
13
+ LCG_SEED = 0x517CC1B727220A95
14
+ LCG_MUL = 6364136223846793005
15
+ LCG_INC = 1442695040888963407
16
+
17
+
18
+ def _assert_num_perm(num_perm: int) -> None:
19
+ if num_perm < 1:
20
+ raise ValueError("[ERR_MINHASH_PERM] num_perm must be > 0")
21
+
22
+
23
+ def _generate_hash_params(num_perm: int) -> list[tuple[int, int]]:
24
+ state = LCG_SEED
25
+ params: list[tuple[int, int]] = []
26
+ for _ in range(num_perm):
27
+ state = (state * LCG_MUL + LCG_INC) & MASK64
28
+ a = state % MERSENNE_61
29
+ if a == 0: # pragma: no cover — defensive; LCG never yields a multiple of p
30
+ a = 1
31
+ state = (state * LCG_MUL + LCG_INC) & MASK64
32
+ b = state % MERSENNE_61
33
+ params.append((a, b))
34
+ return params
35
+
36
+
37
+ def _universal_hash(a: int, b: int, x: int) -> int:
38
+ return (a * (x % MERSENNE_61) + b) % MERSENNE_61
39
+
40
+
41
+ class MinHasher:
42
+ """MinHash signature generator with a deterministic permutation family."""
43
+
44
+ __slots__ = ("_num_perm", "_params", "signature")
45
+
46
+ def __init__(self, num_perm: int = 128) -> None:
47
+ _assert_num_perm(num_perm)
48
+ self._num_perm = num_perm
49
+ self._params = _generate_hash_params(num_perm)
50
+ self.signature = [MASK64] * num_perm
51
+
52
+ @property
53
+ def num_perm(self) -> int:
54
+ return self._num_perm
55
+
56
+ def add(self, token: str) -> None:
57
+ """Fold one token into the signature."""
58
+ h = fnv1a64(token.encode("utf-8"))
59
+ for i in range(self._num_perm):
60
+ a, b = self._params[i]
61
+ val = _universal_hash(a, b, h)
62
+ self.signature[i] = min(self.signature[i], val)
63
+
64
+ def add_tokens(self, tokens: Sequence[str]) -> None:
65
+ """Fold a sequence of tokens into the signature (order-independent)."""
66
+ for t in tokens:
67
+ self.add(t)
68
+
69
+
70
+ def jaccard_estimate(sig_a: Sequence[int], sig_b: Sequence[int]) -> float:
71
+ """Estimated Jaccard = matched positions / signature length."""
72
+ if len(sig_a) != len(sig_b):
73
+ raise ValueError("[ERR_JACCARD_LEN] signatures must have same length")
74
+ if len(sig_a) == 0:
75
+ raise ValueError("[ERR_JACCARD_EMPTY] signatures must not be empty")
76
+ matches = sum(1 for a, b in zip(sig_a, sig_b, strict=False) if a == b)
77
+ return matches / len(sig_a)
@@ -0,0 +1,134 @@
1
+ """SHA-256 (FIPS 180-4) — pure implementation.
2
+
3
+ Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md; vectors in
4
+ vectors/lomboksimhash-vectors-v2.json.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import struct
10
+
11
+ _MASK32 = 0xFFFFFFFF
12
+
13
+ _K_SHA = [
14
+ 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
15
+ 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
16
+ 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
17
+ 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
18
+ 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
19
+ 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
20
+ 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
21
+ 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
22
+ 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
23
+ 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
24
+ 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
25
+ 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
26
+ 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
27
+ 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
28
+ 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
29
+ 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
30
+ ]
31
+
32
+ _H_INIT = [
33
+ 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
34
+ 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19,
35
+ ]
36
+
37
+
38
+ def _rotr32(x: int, n: int) -> int:
39
+ return ((x >> n) | (x << (32 - n))) & _MASK32
40
+
41
+
42
+ def _sha256_compress(state: list[int], block: bytes) -> None:
43
+ w = [0] * 64
44
+ for i in range(16):
45
+ w[i] = struct.unpack(">I", block[i * 4:i * 4 + 4])[0]
46
+ for i in range(16, 64):
47
+ s0 = _rotr32(w[i - 15], 7) ^ _rotr32(w[i - 15], 18) ^ (w[i - 15] >> 3)
48
+ s1 = _rotr32(w[i - 2], 17) ^ _rotr32(w[i - 2], 19) ^ (w[i - 2] >> 10)
49
+ w[i] = (w[i - 16] + s0 + w[i - 7] + s1) & _MASK32
50
+
51
+ a, b, c, d, e, f, g, h = state
52
+
53
+ for i in range(64):
54
+ s1 = _rotr32(e, 6) ^ _rotr32(e, 11) ^ _rotr32(e, 25)
55
+ ch = (e & f) ^ ((~e & _MASK32) & g)
56
+ temp1 = (h + s1 + ch + _K_SHA[i] + w[i]) & _MASK32
57
+ s0 = _rotr32(a, 2) ^ _rotr32(a, 13) ^ _rotr32(a, 22)
58
+ maj = (a & b) ^ (a & c) ^ (b & c)
59
+ temp2 = (s0 + maj) & _MASK32
60
+
61
+ h = g
62
+ g = f
63
+ f = e
64
+ e = (d + temp1) & _MASK32
65
+ d = c
66
+ c = b
67
+ b = a
68
+ a = (temp1 + temp2) & _MASK32
69
+
70
+ for i, v in enumerate((a, b, c, d, e, f, g, h)):
71
+ state[i] = (state[i] + v) & _MASK32
72
+
73
+
74
+ class Sha256State:
75
+ """Incremental SHA-256 hasher (streaming)."""
76
+
77
+ __slots__ = ("_state", "_buffer", "_total_len")
78
+
79
+ def __init__(self) -> None:
80
+ self._state = list(_H_INIT)
81
+ self._buffer = bytearray()
82
+ self._total_len = 0
83
+
84
+ def update(self, data: bytes | bytearray | memoryview) -> None:
85
+ self._total_len += len(data)
86
+ self._buffer.extend(data)
87
+ while len(self._buffer) >= 64:
88
+ block = bytes(self._buffer[:64])
89
+ del self._buffer[:64]
90
+ _sha256_compress(self._state, block)
91
+
92
+ def digest(self) -> bytes:
93
+ """Finalize (without destroying state) and return the 32-byte digest."""
94
+ state = list(self._state)
95
+ buffer = bytearray(self._buffer)
96
+ bit_len = self._total_len * 8
97
+ buffer.append(0x80)
98
+ while len(buffer) % 64 != 56:
99
+ buffer.append(0x00)
100
+ buffer.extend(bit_len.to_bytes(8, "big"))
101
+ for i in range(0, len(buffer), 64):
102
+ _sha256_compress(state, bytes(buffer[i:i + 64]))
103
+ return b"".join(v.to_bytes(4, "big") for v in state)
104
+
105
+ def hexdigest(self) -> str:
106
+ return self.digest().hex()
107
+
108
+ def finalize(self) -> bytes:
109
+ """Finalize and return the 32-byte digest."""
110
+ return self.digest()
111
+
112
+
113
+ def sha256(data: bytes | str) -> str:
114
+ """Compute SHA-256 of bytes or UTF-8 text. Returns lowercase hex string."""
115
+ if isinstance(data, str):
116
+ data = data.encode("utf-8")
117
+ state = list(_H_INIT)
118
+ bit_len = len(data) * 8
119
+
120
+ padded = bytearray(data)
121
+ padded.append(0x80)
122
+ while len(padded) % 64 != 56:
123
+ padded.append(0x00)
124
+ padded.extend(bit_len.to_bytes(8, "big"))
125
+
126
+ for i in range(0, len(padded), 64):
127
+ _sha256_compress(state, bytes(padded[i:i + 64]))
128
+
129
+ return "".join(f"{v:08x}" for v in state)
130
+
131
+
132
+ def to_hex(digest: bytes) -> str:
133
+ """Convert digest bytes to lowercase hex string."""
134
+ return digest.hex()
@@ -0,0 +1,76 @@
1
+ """Shingling — character/word n-gram generation (streaming).
2
+
3
+ Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from ._core import tokenize
9
+
10
+ _SHINGLE_N_MSG = "[ERR_SHINGLE_N] shingle size must be > 0"
11
+
12
+
13
+ def _assert_n(n: int) -> None:
14
+ if n < 1:
15
+ raise ValueError(_SHINGLE_N_MSG)
16
+
17
+
18
+ class Shingler:
19
+ """Streaming shingle generator.
20
+
21
+ Character mode slices Unicode code points; word mode slices tokens
22
+ produced by the SPEC tokenizer and joins them with a single U+0020.
23
+ finish() returns unique shingles sorted by code point.
24
+ """
25
+
26
+ __slots__ = ("_mode", "_n", "_buffer")
27
+
28
+ def __init__(self, mode: str = "character", n: int = 3) -> None:
29
+ _assert_n(n)
30
+ if mode not in ("character", "word"):
31
+ raise ValueError("mode must be 'character' or 'word'")
32
+ self._mode = mode
33
+ self._n = n
34
+ self._buffer = ""
35
+
36
+ @property
37
+ def mode(self) -> str:
38
+ return self._mode
39
+
40
+ @property
41
+ def n(self) -> int:
42
+ return self._n
43
+
44
+ def feed(self, text: str) -> None:
45
+ self._buffer += text
46
+
47
+ def finish(self) -> list[str]:
48
+ """Return sorted unique shingles."""
49
+ if self._mode == "character":
50
+ chars = list(self._buffer)
51
+ count = len(chars)
52
+ return sorted(
53
+ {"".join(chars[i:i + self._n]) for i in range(count - self._n + 1)}
54
+ )
55
+ words = tokenize(self._buffer)
56
+ count = len(words)
57
+ return sorted(
58
+ {" ".join(words[i:i + self._n]) for i in range(count - self._n + 1)}
59
+ )
60
+
61
+ def feed_finish(self, text: str) -> list[str]:
62
+ """Convenience: feed(text) then finish()."""
63
+ self.feed(text)
64
+ return self.finish()
65
+
66
+
67
+ def char_shingles(text: str, n: int) -> list[str]:
68
+ """Convenience: sorted unique character n-grams (code points)."""
69
+ _assert_n(n)
70
+ return Shingler("character", n).feed_finish(text)
71
+
72
+
73
+ def word_shingles(text: str, n: int) -> list[str]:
74
+ """Convenience: sorted unique word n-grams."""
75
+ _assert_n(n)
76
+ return Shingler("word", n).feed_finish(text)
@@ -0,0 +1,77 @@
1
+ """SimHash — 64/128-bit locality-sensitive fingerprinting via FNV-1a.
2
+
3
+ Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from collections.abc import Sequence
9
+
10
+ from ._core import fnv1a64, tokenize
11
+
12
+
13
+ def simhash64(tokens: Sequence[str]) -> int:
14
+ """64-bit SimHash over token list. Bit i set iff weight[i] > 0 (LSB first)."""
15
+ v = [0] * 64
16
+ for token in tokens:
17
+ h = fnv1a64(token.encode("utf-8"))
18
+ for i in range(64):
19
+ if (h >> i) & 1:
20
+ v[i] += 1
21
+ else:
22
+ v[i] -= 1
23
+ result = 0
24
+ for i in range(64):
25
+ if v[i] > 0:
26
+ result |= 1 << i
27
+ return result
28
+
29
+
30
+ def simhash64_text(text: str) -> int:
31
+ """Tokenize by the SPEC whitespace set, then simhash64."""
32
+ return simhash64(tokenize(text))
33
+
34
+
35
+ def simhash128(tokens: Sequence[str]) -> tuple[int, int]:
36
+ """128-bit SimHash. Returns (hi, lo).
37
+
38
+ lo = simhash of fnv1a64(utf8(token)); hi = simhash of
39
+ fnv1a64(utf8(token) || 0xFF).
40
+ """
41
+ v = [0] * 128
42
+ for token in tokens:
43
+ raw = token.encode("utf-8")
44
+ h1 = fnv1a64(raw)
45
+ h2 = fnv1a64(raw + b"\xff")
46
+ for i in range(64):
47
+ if (h1 >> i) & 1:
48
+ v[i] += 1
49
+ else:
50
+ v[i] -= 1
51
+ if (h2 >> i) & 1:
52
+ v[64 + i] += 1
53
+ else:
54
+ v[64 + i] -= 1
55
+ lo = 0
56
+ hi = 0
57
+ for i in range(64):
58
+ if v[i] > 0:
59
+ lo |= 1 << i
60
+ if v[64 + i] > 0:
61
+ hi |= 1 << i
62
+ return (hi, lo)
63
+
64
+
65
+ def simhash128_text(text: str) -> tuple[int, int]:
66
+ """Tokenize by the SPEC whitespace set, then simhash128."""
67
+ return simhash128(tokenize(text))
68
+
69
+
70
+ def hamming64(a: int, b: int) -> int:
71
+ """Hamming distance between two 64-bit values."""
72
+ return bin(a ^ b).count("1")
73
+
74
+
75
+ def hamming128(a: tuple[int, int], b: tuple[int, int]) -> int:
76
+ """Hamming distance between two 128-bit (hi, lo) pairs."""
77
+ return bin(a[0] ^ b[0]).count("1") + bin(a[1] ^ b[1]).count("1")
@@ -0,0 +1,47 @@
1
+ Metadata-Version: 2.4
2
+ Name: lomboksimhash
3
+ Version: 0.1.0
4
+ Summary: Locality-sensitive hashing for near-duplicate document detection. SimHash, MinHash, LSH, SHA-256, and shingling.
5
+ Author-email: CodingLombok <codinglombok@gmail.com>
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/codinglombok/LombokSimHash
8
+ Project-URL: Repository, https://github.com/codinglombok/LombokSimHash.git
9
+ Project-URL: Changelog, https://github.com/codinglombok/LombokSimHash/blob/main/changelog.md
10
+ Keywords: simhash,minhash,lsh,deduplication,hashing,locality-sensitive-hashing,near-duplicate,lombok
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: License :: OSI Approved :: Apache Software License
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Text Processing :: Indexing
20
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
21
+ Requires-Python: >=3.10
22
+ Description-Content-Type: text/markdown
23
+ Provides-Extra: dev
24
+ Requires-Dist: pytest>=8.0; extra == "dev"
25
+ Requires-Dist: pytest-cov>=5.0; extra == "dev"
26
+ Requires-Dist: ruff>=0.8; extra == "dev"
27
+ Requires-Dist: mypy>=1.13; extra == "dev"
28
+
29
+ # LombokSimHash (Python)
30
+
31
+ Near-duplicate document detection for RAG pipelines — SimHash, MinHash, LSH, SHA-256, and shingling. Pure Python, zero dependencies.
32
+
33
+ ```bash
34
+ pip install lomboksimhash
35
+ ```
36
+
37
+ ```python
38
+ from lomboksimhash import simhash64_text, hamming64, MinHasher, jaccard_estimate, LshIndex
39
+
40
+ a = simhash64_text("the quick brown fox jumps over the lazy dog")
41
+ b = simhash64_text("the quick brown cat jumps over the lazy dog")
42
+ print(hamming64(a, b)) # 8
43
+ ```
44
+
45
+ Part of [LombokRAGFrameworks](https://github.com/codinglombok/LombokSimHash). Full docs in the [main repository](https://github.com/codinglombok/LombokSimHash).
46
+
47
+ License: Apache-2.0
@@ -0,0 +1,11 @@
1
+ lomboksimhash/__init__.py,sha256=4WPYXjQwAqTNv9T-dQPd00JmB0lB5OhCN7My-hru33M,1245
2
+ lomboksimhash/_core.py,sha256=PQdvZo72mXbARXx6tppk6CYrsugPO2m3dO3jPowcMXg,1321
3
+ lomboksimhash/_lsh.py,sha256=jsdgPH_W-23DIpbGza-C5re462-ZroiitWiyT0ITt0w,3195
4
+ lomboksimhash/_minhash.py,sha256=snafDmOGGvhOC_fqe2JlT5PUyrCxGsNdV8tlnmxliI4,2500
5
+ lomboksimhash/_sha256.py,sha256=71dexYiK_alht122_ytMl8VIMtULemP94WTYWEvpAWQ,4264
6
+ lomboksimhash/_shingle.py,sha256=CUR6C_hyyNb0cagtIQcjx5g7mhNgOd0p6Rvj3hJUhng,2135
7
+ lomboksimhash/_simhash.py,sha256=wwV-imSbS4OxDVFILeY-GCKp3U9GSMWvyOC7SpAgdIo,2055
8
+ lomboksimhash-0.1.0.dist-info/METADATA,sha256=w-XEOq9nnlKLKigkBw2b6VbXEEKDXAJ8eN9cEdgSNnw,2016
9
+ lomboksimhash-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
10
+ lomboksimhash-0.1.0.dist-info/top_level.txt,sha256=wlZB4oipM_rnpw8dTXpvBJBPed1hVSerqgnHVGVI1ss,14
11
+ lomboksimhash-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (84.2.1)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1 @@
1
+ lomboksimhash