lomboksimhash 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lomboksimhash/__init__.py +48 -0
- lomboksimhash/_core.py +50 -0
- lomboksimhash/_lsh.py +95 -0
- lomboksimhash/_minhash.py +77 -0
- lomboksimhash/_sha256.py +134 -0
- lomboksimhash/_shingle.py +76 -0
- lomboksimhash/_simhash.py +77 -0
- lomboksimhash-0.1.0.dist-info/METADATA +47 -0
- lomboksimhash-0.1.0.dist-info/RECORD +11 -0
- lomboksimhash-0.1.0.dist-info/WHEEL +5 -0
- lomboksimhash-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""LombokSimHash — Python Port.
|
|
2
|
+
|
|
3
|
+
Near-duplicate document detection for RAG pipelines:
|
|
4
|
+
SHA-256, FNV-1a 64, SimHash 64/128, MinHash, LSH, shingling.
|
|
5
|
+
|
|
6
|
+
Zero dependencies. Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md;
|
|
7
|
+
all ports are validated against vectors/lomboksimhash-vectors-v2.json.
|
|
8
|
+
|
|
9
|
+
Part of LombokRAGFrameworks (Lombok Ecosystem).
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from ._core import FNV64_OFFSET, FNV64_PRIME, MASK64, WHITESPACE, fnv1a64, to_hex64, tokenize
|
|
16
|
+
from ._lsh import LshIndex, band_hash
|
|
17
|
+
from ._minhash import MinHasher, jaccard_estimate
|
|
18
|
+
from ._sha256 import Sha256State, sha256, to_hex
|
|
19
|
+
from ._shingle import Shingler, char_shingles, word_shingles
|
|
20
|
+
from ._simhash import hamming64, hamming128, simhash64, simhash64_text, simhash128, simhash128_text
|
|
21
|
+
|
|
22
|
+
__version__ = "0.1.0"
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"MASK64",
|
|
26
|
+
"FNV64_OFFSET",
|
|
27
|
+
"FNV64_PRIME",
|
|
28
|
+
"WHITESPACE",
|
|
29
|
+
"Sha256State",
|
|
30
|
+
"LshIndex",
|
|
31
|
+
"MinHasher",
|
|
32
|
+
"Shingler",
|
|
33
|
+
"band_hash",
|
|
34
|
+
"char_shingles",
|
|
35
|
+
"word_shingles",
|
|
36
|
+
"fnv1a64",
|
|
37
|
+
"hamming128",
|
|
38
|
+
"hamming64",
|
|
39
|
+
"jaccard_estimate",
|
|
40
|
+
"sha256",
|
|
41
|
+
"simhash128",
|
|
42
|
+
"simhash128_text",
|
|
43
|
+
"simhash64",
|
|
44
|
+
"simhash64_text",
|
|
45
|
+
"to_hex",
|
|
46
|
+
"to_hex64",
|
|
47
|
+
"tokenize",
|
|
48
|
+
]
|
lomboksimhash/_core.py
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""FNV-1a 64-bit and shared helpers (tokenization, hex formatting).
|
|
2
|
+
|
|
3
|
+
Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
FNV64_OFFSET = 0xCBF29CE484222325
|
|
9
|
+
FNV64_PRIME = 0x100000001B3
|
|
10
|
+
MASK64 = (1 << 64) - 1
|
|
11
|
+
|
|
12
|
+
# SPEC whitespace set = Unicode White_Space property (exact, cross-language).
|
|
13
|
+
WHITESPACE = frozenset(
|
|
14
|
+
"\t\n\x0b\x0c\r \u0085\u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005"
|
|
15
|
+
"\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000"
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def fnv1a64(data: bytes) -> int:
|
|
20
|
+
"""FNV-1a 64-bit hash. offset 0xcbf29ce484222325, prime 0x100000001b3."""
|
|
21
|
+
h = FNV64_OFFSET
|
|
22
|
+
for b in data:
|
|
23
|
+
h ^= b
|
|
24
|
+
h = (h * FNV64_PRIME) & MASK64
|
|
25
|
+
return h
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def to_hex64(value: int) -> str:
|
|
29
|
+
"""Format a u64 as 16 lowercase hex chars."""
|
|
30
|
+
return f"{value:016x}"
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def tokenize(text: str) -> list[str]:
|
|
34
|
+
"""Split text on runs of the SPEC whitespace set; drop empty parts."""
|
|
35
|
+
return [t for t in _split_ws(text) if t]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _split_ws(text: str) -> list[str]:
|
|
39
|
+
parts: list[str] = []
|
|
40
|
+
cur: list[str] = []
|
|
41
|
+
for ch in text:
|
|
42
|
+
if ch in WHITESPACE:
|
|
43
|
+
if cur:
|
|
44
|
+
parts.append("".join(cur))
|
|
45
|
+
cur = []
|
|
46
|
+
else:
|
|
47
|
+
cur.append(ch)
|
|
48
|
+
if cur:
|
|
49
|
+
parts.append("".join(cur))
|
|
50
|
+
return parts
|
lomboksimhash/_lsh.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""LSH — band+row locality-sensitive hashing over MinHash signatures.
|
|
2
|
+
|
|
3
|
+
Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
from ._core import FNV64_OFFSET, FNV64_PRIME, MASK64
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _assert_config(bands: int, rows: int) -> None:
|
|
14
|
+
if bands < 1 or rows < 1:
|
|
15
|
+
raise ValueError("[ERR_LSH_CONFIG] bands and rows must be > 0")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _assert_sig_len(signature: Sequence[int], expected: int) -> None:
|
|
19
|
+
if len(signature) != expected:
|
|
20
|
+
raise ValueError(
|
|
21
|
+
f"[ERR_SIG_LEN] signature length must be bands * rows = {expected}"
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def band_hash(band_values: Sequence[int]) -> int:
|
|
26
|
+
"""FNV-1a 64 over the little-endian bytes of each u64 in the band."""
|
|
27
|
+
h = FNV64_OFFSET
|
|
28
|
+
for val in band_values:
|
|
29
|
+
for byte_idx in range(8):
|
|
30
|
+
byte = (val >> (byte_idx * 8)) & 0xFF
|
|
31
|
+
h ^= byte
|
|
32
|
+
h = (h * FNV64_PRIME) & MASK64
|
|
33
|
+
return h
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class LshIndex:
|
|
37
|
+
"""LSH index (band+row) over MinHash signatures.
|
|
38
|
+
|
|
39
|
+
query() returns de-duplicated doc IDs sorted ascending by code point.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
__slots__ = ("_bands", "_rows", "_buckets")
|
|
43
|
+
|
|
44
|
+
def __init__(self, bands: int, rows: int) -> None:
|
|
45
|
+
_assert_config(bands, rows)
|
|
46
|
+
self._bands = bands
|
|
47
|
+
self._rows = rows
|
|
48
|
+
self._buckets: list[dict[int, list[str]]] = [{} for _ in range(bands)]
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def bands(self) -> int:
|
|
52
|
+
return self._bands
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def rows(self) -> int:
|
|
56
|
+
return self._rows
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def signature_len(self) -> int:
|
|
60
|
+
return self._bands * self._rows
|
|
61
|
+
|
|
62
|
+
def threshold(self) -> float:
|
|
63
|
+
"""Approximate Jaccard threshold (1/bands)^(1/rows). Informational."""
|
|
64
|
+
base = 1.0 / self._bands
|
|
65
|
+
expo = 1.0 / self._rows
|
|
66
|
+
return float(base ** expo)
|
|
67
|
+
|
|
68
|
+
def insert(self, doc_id: str, signature: Sequence[int]) -> None:
|
|
69
|
+
"""Insert a document signature into every band."""
|
|
70
|
+
_assert_sig_len(signature, self.signature_len)
|
|
71
|
+
for b in range(self._bands):
|
|
72
|
+
start = b * self._rows
|
|
73
|
+
key = band_hash(signature[start:start + self._rows])
|
|
74
|
+
self._buckets[b].setdefault(key, []).append(doc_id)
|
|
75
|
+
|
|
76
|
+
def query(self, signature: Sequence[int]) -> list[str]:
|
|
77
|
+
"""Return sorted unique candidate doc IDs for the signature."""
|
|
78
|
+
_assert_sig_len(signature, self.signature_len)
|
|
79
|
+
candidates: set[str] = set()
|
|
80
|
+
for b in range(self._bands):
|
|
81
|
+
start = b * self._rows
|
|
82
|
+
key = band_hash(signature[start:start + self._rows])
|
|
83
|
+
candidates.update(self._buckets[b].get(key, ()))
|
|
84
|
+
return sorted(candidates)
|
|
85
|
+
|
|
86
|
+
def remove(self, doc_id: str, signature: Sequence[int]) -> None:
|
|
87
|
+
"""Remove a doc ID from every band bucket touched by the signature."""
|
|
88
|
+
_assert_sig_len(signature, self.signature_len)
|
|
89
|
+
for b in range(self._bands):
|
|
90
|
+
start = b * self._rows
|
|
91
|
+
key = band_hash(signature[start:start + self._rows])
|
|
92
|
+
if key in self._buckets[b]:
|
|
93
|
+
self._buckets[b][key] = [
|
|
94
|
+
d for d in self._buckets[b][key] if d != doc_id
|
|
95
|
+
]
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""MinHash — Jaccard estimation via deterministic k-permutation family.
|
|
2
|
+
|
|
3
|
+
Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
from ._core import MASK64, fnv1a64
|
|
11
|
+
|
|
12
|
+
MERSENNE_61 = (1 << 61) - 1
|
|
13
|
+
LCG_SEED = 0x517CC1B727220A95
|
|
14
|
+
LCG_MUL = 6364136223846793005
|
|
15
|
+
LCG_INC = 1442695040888963407
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _assert_num_perm(num_perm: int) -> None:
|
|
19
|
+
if num_perm < 1:
|
|
20
|
+
raise ValueError("[ERR_MINHASH_PERM] num_perm must be > 0")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _generate_hash_params(num_perm: int) -> list[tuple[int, int]]:
|
|
24
|
+
state = LCG_SEED
|
|
25
|
+
params: list[tuple[int, int]] = []
|
|
26
|
+
for _ in range(num_perm):
|
|
27
|
+
state = (state * LCG_MUL + LCG_INC) & MASK64
|
|
28
|
+
a = state % MERSENNE_61
|
|
29
|
+
if a == 0: # pragma: no cover — defensive; LCG never yields a multiple of p
|
|
30
|
+
a = 1
|
|
31
|
+
state = (state * LCG_MUL + LCG_INC) & MASK64
|
|
32
|
+
b = state % MERSENNE_61
|
|
33
|
+
params.append((a, b))
|
|
34
|
+
return params
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _universal_hash(a: int, b: int, x: int) -> int:
|
|
38
|
+
return (a * (x % MERSENNE_61) + b) % MERSENNE_61
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class MinHasher:
|
|
42
|
+
"""MinHash signature generator with a deterministic permutation family."""
|
|
43
|
+
|
|
44
|
+
__slots__ = ("_num_perm", "_params", "signature")
|
|
45
|
+
|
|
46
|
+
def __init__(self, num_perm: int = 128) -> None:
|
|
47
|
+
_assert_num_perm(num_perm)
|
|
48
|
+
self._num_perm = num_perm
|
|
49
|
+
self._params = _generate_hash_params(num_perm)
|
|
50
|
+
self.signature = [MASK64] * num_perm
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def num_perm(self) -> int:
|
|
54
|
+
return self._num_perm
|
|
55
|
+
|
|
56
|
+
def add(self, token: str) -> None:
|
|
57
|
+
"""Fold one token into the signature."""
|
|
58
|
+
h = fnv1a64(token.encode("utf-8"))
|
|
59
|
+
for i in range(self._num_perm):
|
|
60
|
+
a, b = self._params[i]
|
|
61
|
+
val = _universal_hash(a, b, h)
|
|
62
|
+
self.signature[i] = min(self.signature[i], val)
|
|
63
|
+
|
|
64
|
+
def add_tokens(self, tokens: Sequence[str]) -> None:
|
|
65
|
+
"""Fold a sequence of tokens into the signature (order-independent)."""
|
|
66
|
+
for t in tokens:
|
|
67
|
+
self.add(t)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def jaccard_estimate(sig_a: Sequence[int], sig_b: Sequence[int]) -> float:
|
|
71
|
+
"""Estimated Jaccard = matched positions / signature length."""
|
|
72
|
+
if len(sig_a) != len(sig_b):
|
|
73
|
+
raise ValueError("[ERR_JACCARD_LEN] signatures must have same length")
|
|
74
|
+
if len(sig_a) == 0:
|
|
75
|
+
raise ValueError("[ERR_JACCARD_EMPTY] signatures must not be empty")
|
|
76
|
+
matches = sum(1 for a, b in zip(sig_a, sig_b, strict=False) if a == b)
|
|
77
|
+
return matches / len(sig_a)
|
lomboksimhash/_sha256.py
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""SHA-256 (FIPS 180-4) — pure implementation.
|
|
2
|
+
|
|
3
|
+
Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md; vectors in
|
|
4
|
+
vectors/lomboksimhash-vectors-v2.json.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import struct
|
|
10
|
+
|
|
11
|
+
_MASK32 = 0xFFFFFFFF
|
|
12
|
+
|
|
13
|
+
_K_SHA = [
|
|
14
|
+
0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5,
|
|
15
|
+
0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
|
|
16
|
+
0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3,
|
|
17
|
+
0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
|
|
18
|
+
0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC,
|
|
19
|
+
0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
|
|
20
|
+
0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7,
|
|
21
|
+
0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
|
|
22
|
+
0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13,
|
|
23
|
+
0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
|
|
24
|
+
0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3,
|
|
25
|
+
0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
|
|
26
|
+
0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5,
|
|
27
|
+
0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
|
|
28
|
+
0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208,
|
|
29
|
+
0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
_H_INIT = [
|
|
33
|
+
0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
|
|
34
|
+
0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19,
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _rotr32(x: int, n: int) -> int:
|
|
39
|
+
return ((x >> n) | (x << (32 - n))) & _MASK32
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _sha256_compress(state: list[int], block: bytes) -> None:
|
|
43
|
+
w = [0] * 64
|
|
44
|
+
for i in range(16):
|
|
45
|
+
w[i] = struct.unpack(">I", block[i * 4:i * 4 + 4])[0]
|
|
46
|
+
for i in range(16, 64):
|
|
47
|
+
s0 = _rotr32(w[i - 15], 7) ^ _rotr32(w[i - 15], 18) ^ (w[i - 15] >> 3)
|
|
48
|
+
s1 = _rotr32(w[i - 2], 17) ^ _rotr32(w[i - 2], 19) ^ (w[i - 2] >> 10)
|
|
49
|
+
w[i] = (w[i - 16] + s0 + w[i - 7] + s1) & _MASK32
|
|
50
|
+
|
|
51
|
+
a, b, c, d, e, f, g, h = state
|
|
52
|
+
|
|
53
|
+
for i in range(64):
|
|
54
|
+
s1 = _rotr32(e, 6) ^ _rotr32(e, 11) ^ _rotr32(e, 25)
|
|
55
|
+
ch = (e & f) ^ ((~e & _MASK32) & g)
|
|
56
|
+
temp1 = (h + s1 + ch + _K_SHA[i] + w[i]) & _MASK32
|
|
57
|
+
s0 = _rotr32(a, 2) ^ _rotr32(a, 13) ^ _rotr32(a, 22)
|
|
58
|
+
maj = (a & b) ^ (a & c) ^ (b & c)
|
|
59
|
+
temp2 = (s0 + maj) & _MASK32
|
|
60
|
+
|
|
61
|
+
h = g
|
|
62
|
+
g = f
|
|
63
|
+
f = e
|
|
64
|
+
e = (d + temp1) & _MASK32
|
|
65
|
+
d = c
|
|
66
|
+
c = b
|
|
67
|
+
b = a
|
|
68
|
+
a = (temp1 + temp2) & _MASK32
|
|
69
|
+
|
|
70
|
+
for i, v in enumerate((a, b, c, d, e, f, g, h)):
|
|
71
|
+
state[i] = (state[i] + v) & _MASK32
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Sha256State:
|
|
75
|
+
"""Incremental SHA-256 hasher (streaming)."""
|
|
76
|
+
|
|
77
|
+
__slots__ = ("_state", "_buffer", "_total_len")
|
|
78
|
+
|
|
79
|
+
def __init__(self) -> None:
|
|
80
|
+
self._state = list(_H_INIT)
|
|
81
|
+
self._buffer = bytearray()
|
|
82
|
+
self._total_len = 0
|
|
83
|
+
|
|
84
|
+
def update(self, data: bytes | bytearray | memoryview) -> None:
|
|
85
|
+
self._total_len += len(data)
|
|
86
|
+
self._buffer.extend(data)
|
|
87
|
+
while len(self._buffer) >= 64:
|
|
88
|
+
block = bytes(self._buffer[:64])
|
|
89
|
+
del self._buffer[:64]
|
|
90
|
+
_sha256_compress(self._state, block)
|
|
91
|
+
|
|
92
|
+
def digest(self) -> bytes:
|
|
93
|
+
"""Finalize (without destroying state) and return the 32-byte digest."""
|
|
94
|
+
state = list(self._state)
|
|
95
|
+
buffer = bytearray(self._buffer)
|
|
96
|
+
bit_len = self._total_len * 8
|
|
97
|
+
buffer.append(0x80)
|
|
98
|
+
while len(buffer) % 64 != 56:
|
|
99
|
+
buffer.append(0x00)
|
|
100
|
+
buffer.extend(bit_len.to_bytes(8, "big"))
|
|
101
|
+
for i in range(0, len(buffer), 64):
|
|
102
|
+
_sha256_compress(state, bytes(buffer[i:i + 64]))
|
|
103
|
+
return b"".join(v.to_bytes(4, "big") for v in state)
|
|
104
|
+
|
|
105
|
+
def hexdigest(self) -> str:
|
|
106
|
+
return self.digest().hex()
|
|
107
|
+
|
|
108
|
+
def finalize(self) -> bytes:
|
|
109
|
+
"""Finalize and return the 32-byte digest."""
|
|
110
|
+
return self.digest()
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def sha256(data: bytes | str) -> str:
|
|
114
|
+
"""Compute SHA-256 of bytes or UTF-8 text. Returns lowercase hex string."""
|
|
115
|
+
if isinstance(data, str):
|
|
116
|
+
data = data.encode("utf-8")
|
|
117
|
+
state = list(_H_INIT)
|
|
118
|
+
bit_len = len(data) * 8
|
|
119
|
+
|
|
120
|
+
padded = bytearray(data)
|
|
121
|
+
padded.append(0x80)
|
|
122
|
+
while len(padded) % 64 != 56:
|
|
123
|
+
padded.append(0x00)
|
|
124
|
+
padded.extend(bit_len.to_bytes(8, "big"))
|
|
125
|
+
|
|
126
|
+
for i in range(0, len(padded), 64):
|
|
127
|
+
_sha256_compress(state, bytes(padded[i:i + 64]))
|
|
128
|
+
|
|
129
|
+
return "".join(f"{v:08x}" for v in state)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def to_hex(digest: bytes) -> str:
|
|
133
|
+
"""Convert digest bytes to lowercase hex string."""
|
|
134
|
+
return digest.hex()
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Shingling — character/word n-gram generation (streaming).
|
|
2
|
+
|
|
3
|
+
Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from ._core import tokenize
|
|
9
|
+
|
|
10
|
+
_SHINGLE_N_MSG = "[ERR_SHINGLE_N] shingle size must be > 0"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _assert_n(n: int) -> None:
|
|
14
|
+
if n < 1:
|
|
15
|
+
raise ValueError(_SHINGLE_N_MSG)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Shingler:
|
|
19
|
+
"""Streaming shingle generator.
|
|
20
|
+
|
|
21
|
+
Character mode slices Unicode code points; word mode slices tokens
|
|
22
|
+
produced by the SPEC tokenizer and joins them with a single U+0020.
|
|
23
|
+
finish() returns unique shingles sorted by code point.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
__slots__ = ("_mode", "_n", "_buffer")
|
|
27
|
+
|
|
28
|
+
def __init__(self, mode: str = "character", n: int = 3) -> None:
|
|
29
|
+
_assert_n(n)
|
|
30
|
+
if mode not in ("character", "word"):
|
|
31
|
+
raise ValueError("mode must be 'character' or 'word'")
|
|
32
|
+
self._mode = mode
|
|
33
|
+
self._n = n
|
|
34
|
+
self._buffer = ""
|
|
35
|
+
|
|
36
|
+
@property
|
|
37
|
+
def mode(self) -> str:
|
|
38
|
+
return self._mode
|
|
39
|
+
|
|
40
|
+
@property
|
|
41
|
+
def n(self) -> int:
|
|
42
|
+
return self._n
|
|
43
|
+
|
|
44
|
+
def feed(self, text: str) -> None:
|
|
45
|
+
self._buffer += text
|
|
46
|
+
|
|
47
|
+
def finish(self) -> list[str]:
|
|
48
|
+
"""Return sorted unique shingles."""
|
|
49
|
+
if self._mode == "character":
|
|
50
|
+
chars = list(self._buffer)
|
|
51
|
+
count = len(chars)
|
|
52
|
+
return sorted(
|
|
53
|
+
{"".join(chars[i:i + self._n]) for i in range(count - self._n + 1)}
|
|
54
|
+
)
|
|
55
|
+
words = tokenize(self._buffer)
|
|
56
|
+
count = len(words)
|
|
57
|
+
return sorted(
|
|
58
|
+
{" ".join(words[i:i + self._n]) for i in range(count - self._n + 1)}
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def feed_finish(self, text: str) -> list[str]:
|
|
62
|
+
"""Convenience: feed(text) then finish()."""
|
|
63
|
+
self.feed(text)
|
|
64
|
+
return self.finish()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def char_shingles(text: str, n: int) -> list[str]:
|
|
68
|
+
"""Convenience: sorted unique character n-grams (code points)."""
|
|
69
|
+
_assert_n(n)
|
|
70
|
+
return Shingler("character", n).feed_finish(text)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def word_shingles(text: str, n: int) -> list[str]:
|
|
74
|
+
"""Convenience: sorted unique word n-grams."""
|
|
75
|
+
_assert_n(n)
|
|
76
|
+
return Shingler("word", n).feed_finish(text)
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""SimHash — 64/128-bit locality-sensitive fingerprinting via FNV-1a.
|
|
2
|
+
|
|
3
|
+
Behavior pinned by docs/SPEC_LombokSimHash_0.1.0.md.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from collections.abc import Sequence
|
|
9
|
+
|
|
10
|
+
from ._core import fnv1a64, tokenize
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def simhash64(tokens: Sequence[str]) -> int:
|
|
14
|
+
"""64-bit SimHash over token list. Bit i set iff weight[i] > 0 (LSB first)."""
|
|
15
|
+
v = [0] * 64
|
|
16
|
+
for token in tokens:
|
|
17
|
+
h = fnv1a64(token.encode("utf-8"))
|
|
18
|
+
for i in range(64):
|
|
19
|
+
if (h >> i) & 1:
|
|
20
|
+
v[i] += 1
|
|
21
|
+
else:
|
|
22
|
+
v[i] -= 1
|
|
23
|
+
result = 0
|
|
24
|
+
for i in range(64):
|
|
25
|
+
if v[i] > 0:
|
|
26
|
+
result |= 1 << i
|
|
27
|
+
return result
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def simhash64_text(text: str) -> int:
|
|
31
|
+
"""Tokenize by the SPEC whitespace set, then simhash64."""
|
|
32
|
+
return simhash64(tokenize(text))
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def simhash128(tokens: Sequence[str]) -> tuple[int, int]:
|
|
36
|
+
"""128-bit SimHash. Returns (hi, lo).
|
|
37
|
+
|
|
38
|
+
lo = simhash of fnv1a64(utf8(token)); hi = simhash of
|
|
39
|
+
fnv1a64(utf8(token) || 0xFF).
|
|
40
|
+
"""
|
|
41
|
+
v = [0] * 128
|
|
42
|
+
for token in tokens:
|
|
43
|
+
raw = token.encode("utf-8")
|
|
44
|
+
h1 = fnv1a64(raw)
|
|
45
|
+
h2 = fnv1a64(raw + b"\xff")
|
|
46
|
+
for i in range(64):
|
|
47
|
+
if (h1 >> i) & 1:
|
|
48
|
+
v[i] += 1
|
|
49
|
+
else:
|
|
50
|
+
v[i] -= 1
|
|
51
|
+
if (h2 >> i) & 1:
|
|
52
|
+
v[64 + i] += 1
|
|
53
|
+
else:
|
|
54
|
+
v[64 + i] -= 1
|
|
55
|
+
lo = 0
|
|
56
|
+
hi = 0
|
|
57
|
+
for i in range(64):
|
|
58
|
+
if v[i] > 0:
|
|
59
|
+
lo |= 1 << i
|
|
60
|
+
if v[64 + i] > 0:
|
|
61
|
+
hi |= 1 << i
|
|
62
|
+
return (hi, lo)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def simhash128_text(text: str) -> tuple[int, int]:
|
|
66
|
+
"""Tokenize by the SPEC whitespace set, then simhash128."""
|
|
67
|
+
return simhash128(tokenize(text))
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def hamming64(a: int, b: int) -> int:
|
|
71
|
+
"""Hamming distance between two 64-bit values."""
|
|
72
|
+
return bin(a ^ b).count("1")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def hamming128(a: tuple[int, int], b: tuple[int, int]) -> int:
|
|
76
|
+
"""Hamming distance between two 128-bit (hi, lo) pairs."""
|
|
77
|
+
return bin(a[0] ^ b[0]).count("1") + bin(a[1] ^ b[1]).count("1")
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lomboksimhash
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Locality-sensitive hashing for near-duplicate document detection. SimHash, MinHash, LSH, SHA-256, and shingling.
|
|
5
|
+
Author-email: CodingLombok <codinglombok@gmail.com>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/codinglombok/LombokSimHash
|
|
8
|
+
Project-URL: Repository, https://github.com/codinglombok/LombokSimHash.git
|
|
9
|
+
Project-URL: Changelog, https://github.com/codinglombok/LombokSimHash/blob/main/changelog.md
|
|
10
|
+
Keywords: simhash,minhash,lsh,deduplication,hashing,locality-sensitive-hashing,near-duplicate,lombok
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
25
|
+
Requires-Dist: pytest-cov>=5.0; extra == "dev"
|
|
26
|
+
Requires-Dist: ruff>=0.8; extra == "dev"
|
|
27
|
+
Requires-Dist: mypy>=1.13; extra == "dev"
|
|
28
|
+
|
|
29
|
+
# LombokSimHash (Python)
|
|
30
|
+
|
|
31
|
+
Near-duplicate document detection for RAG pipelines — SimHash, MinHash, LSH, SHA-256, and shingling. Pure Python, zero dependencies.
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install lomboksimhash
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```python
|
|
38
|
+
from lomboksimhash import simhash64_text, hamming64, MinHasher, jaccard_estimate, LshIndex
|
|
39
|
+
|
|
40
|
+
a = simhash64_text("the quick brown fox jumps over the lazy dog")
|
|
41
|
+
b = simhash64_text("the quick brown cat jumps over the lazy dog")
|
|
42
|
+
print(hamming64(a, b)) # 8
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Part of [LombokRAGFrameworks](https://github.com/codinglombok/LombokSimHash). Full docs in the [main repository](https://github.com/codinglombok/LombokSimHash).
|
|
46
|
+
|
|
47
|
+
License: Apache-2.0
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
lomboksimhash/__init__.py,sha256=4WPYXjQwAqTNv9T-dQPd00JmB0lB5OhCN7My-hru33M,1245
|
|
2
|
+
lomboksimhash/_core.py,sha256=PQdvZo72mXbARXx6tppk6CYrsugPO2m3dO3jPowcMXg,1321
|
|
3
|
+
lomboksimhash/_lsh.py,sha256=jsdgPH_W-23DIpbGza-C5re462-ZroiitWiyT0ITt0w,3195
|
|
4
|
+
lomboksimhash/_minhash.py,sha256=snafDmOGGvhOC_fqe2JlT5PUyrCxGsNdV8tlnmxliI4,2500
|
|
5
|
+
lomboksimhash/_sha256.py,sha256=71dexYiK_alht122_ytMl8VIMtULemP94WTYWEvpAWQ,4264
|
|
6
|
+
lomboksimhash/_shingle.py,sha256=CUR6C_hyyNb0cagtIQcjx5g7mhNgOd0p6Rvj3hJUhng,2135
|
|
7
|
+
lomboksimhash/_simhash.py,sha256=wwV-imSbS4OxDVFILeY-GCKp3U9GSMWvyOC7SpAgdIo,2055
|
|
8
|
+
lomboksimhash-0.1.0.dist-info/METADATA,sha256=w-XEOq9nnlKLKigkBw2b6VbXEEKDXAJ8eN9cEdgSNnw,2016
|
|
9
|
+
lomboksimhash-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
10
|
+
lomboksimhash-0.1.0.dist-info/top_level.txt,sha256=wlZB4oipM_rnpw8dTXpvBJBPed1hVSerqgnHVGVI1ss,14
|
|
11
|
+
lomboksimhash-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
lomboksimhash
|