0din-prompt-toolkit 0.9.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. 0din_prompt_toolkit-0.9.3/.gitignore +69 -0
  2. 0din_prompt_toolkit-0.9.3/INSTALL.md +59 -0
  3. 0din_prompt_toolkit-0.9.3/PKG-INFO +229 -0
  4. 0din_prompt_toolkit-0.9.3/README.md +172 -0
  5. 0din_prompt_toolkit-0.9.3/examples/basic_signature.py +63 -0
  6. 0din_prompt_toolkit-0.9.3/examples/cm_lsh_example.py +88 -0
  7. 0din_prompt_toolkit-0.9.3/examples/duplicate_detection.py +94 -0
  8. 0din_prompt_toolkit-0.9.3/examples/sign_text.py +158 -0
  9. 0din_prompt_toolkit-0.9.3/examples/similarity_comparison.py +75 -0
  10. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/__init__.py +126 -0
  11. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/_accel.py +93 -0
  12. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/cm_lsh.py +463 -0
  13. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/error.py +116 -0
  14. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/hasher.py +46 -0
  15. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/hashers/__init__.py +31 -0
  16. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/hashers/lsh.py +56 -0
  17. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/lsh.py +223 -0
  18. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/provider.py +80 -0
  19. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/__init__.py +27 -0
  20. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/model_cache.py +219 -0
  21. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/onnx.py +220 -0
  22. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/openai.py +126 -0
  23. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/py.typed +0 -0
  24. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/sign.py +233 -0
  25. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/__init__.py +74 -0
  26. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/classifier.py +353 -0
  27. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/compare.py +72 -0
  28. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/onnx_classifier.py +378 -0
  29. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/types.py +143 -0
  30. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/__init__.py +38 -0
  31. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/cache.py +325 -0
  32. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/client.py +197 -0
  33. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/compare.py +43 -0
  34. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/types.py +112 -0
  35. 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/types.py +276 -0
  36. 0din_prompt_toolkit-0.9.3/pyproject.toml +166 -0
  37. 0din_prompt_toolkit-0.9.3/tests/test_cm_lsh_vectors.py +118 -0
  38. 0din_prompt_toolkit-0.9.3/tests/test_cross_validation.py +148 -0
  39. 0din_prompt_toolkit-0.9.3/tests/test_error.py +126 -0
  40. 0din_prompt_toolkit-0.9.3/tests/test_hasher.py +63 -0
  41. 0din_prompt_toolkit-0.9.3/tests/test_sign_text.py +172 -0
  42. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_cache.py +97 -0
  43. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_chunking.py +251 -0
  44. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_classifier.py +225 -0
  45. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_compare.py +69 -0
  46. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_integration.py +47 -0
  47. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_onnx_parity.py +398 -0
  48. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_parity.py +142 -0
  49. 0din_prompt_toolkit-0.9.3/tests/test_susfactor_types.py +145 -0
  50. 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_cache.py +177 -0
  51. 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_client.py +396 -0
  52. 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_compare.py +110 -0
  53. 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_fixture.py +136 -0
  54. 0din_prompt_toolkit-0.9.3/tests/test_tokenizer_parity.py +79 -0
  55. 0din_prompt_toolkit-0.9.3/tests/test_types.py +88 -0
  56. 0din_prompt_toolkit-0.9.3/tests/test_vectors.py +179 -0
@@ -0,0 +1,69 @@
1
+ # Build artifacts
2
+ /dist/
3
+
4
+ # Rust
5
+ /packages/rust/target/
6
+ /packages/python-native/target/
7
+ *.rs.bk
8
+
9
+ # Python
10
+ /packages/python/__pycache__/
11
+ /packages/python/*.py[cod]
12
+ /packages/python/*$py.class
13
+ /packages/python/.pytest_cache/
14
+ /packages/python/.coverage
15
+ /packages/python/htmlcov/
16
+ /packages/python/dist/
17
+ /packages/python/build/
18
+ /packages/python/*.egg-info/
19
+ /packages/python/.mypy_cache/
20
+ /packages/python/.ruff_cache/
21
+ /packages/python/.hatch/
22
+
23
+ # TypeScript
24
+ /packages/typescript/node_modules/
25
+ /packages/typescript/dist/
26
+ /packages/typescript/*.tgz
27
+ /packages/typescript/.turbo/
28
+
29
+ # Documentation
30
+ /docs/node_modules/
31
+ /docs/.docusaurus/
32
+ /docs/build/
33
+
34
+ # IDE
35
+ .vscode/
36
+ .idea/
37
+ *.swp
38
+ *.swo
39
+ *~
40
+
41
+ # OS
42
+ .DS_Store
43
+ Thumbs.db
44
+
45
+ # Temporary files
46
+ *.log
47
+ *.tmp
48
+ .env
49
+ .env.local
50
+ demos/cache/*.npz
51
+ demos/cache/*.json
52
+ demos/cache/*.db
53
+
54
+ # OpenCode
55
+ .opencode/
56
+
57
+ # Model files (large ONNX models, don't commit to repo)
58
+ models/*/onnx/*.onnx
59
+ models/*/onnx/*.onnx_data
60
+ models/*/tokenizer.json
61
+ models/*/*.json
62
+ # ...but keep the small metadata config.json (tracked for every model version)
63
+ !models/*/config.json
64
+ docs/static/api/rust/
65
+
66
+ # build/test artifacts (accidentally committed once; keep untracked)
67
+ packages/typescript/coverage/
68
+ *.db-shm
69
+ *.db-wal
@@ -0,0 +1,59 @@
1
+ # Installing odin-prompt-toolkit (Python)
2
+
3
+ ## Requirements
4
+ - Python >= 3.10
5
+
6
+ ## Install from wheel
7
+
8
+ ```bash
9
+ pip install odin_prompt_toolkit-<VERSION>-py3-none-any.whl
10
+ ```
11
+
12
+ Replace `<VERSION>` with the version number (e.g., `0.1.1`).
13
+
14
+ ## Install with optional features
15
+
16
+ ```bash
17
+ # OpenAI embeddings (API-based)
18
+ pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[openai]"
19
+
20
+ # ONNX embeddings (local, no API key needed)
21
+ pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[onnx]"
22
+
23
+ # Confidence Matrix LSH (higher accuracy)
24
+ pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[cm-lsh]"
25
+
26
+ # All features
27
+ pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[all]"
28
+ ```
29
+
30
+ ## Quick Start
31
+
32
+ ```python
33
+ from odin_prompt_toolkit import simhash_lsh_multi, normalize_vector
34
+
35
+ # Your embedding vector (must be L2-normalized)
36
+ vector = [0.5, 0.5, 0.5, 0.5]
37
+ normalized = normalize_vector(vector)
38
+
39
+ # Generate LSH signatures (3 families, 256 bits, 16 bands)
40
+ families = simhash_lsh_multi(normalized)
41
+
42
+ print(f"Signature: {families[0].signature}")
43
+ print(f"Bands: {families[0].bands}")
44
+ ```
45
+
46
+ ## Signature Versions
47
+
48
+ - **V0**: OpenAI text-embedding-3-large (1536 dimensions, API-based)
49
+ - **V1**: 0din-jailbreak-embeddings-small ONNX (1024 dimensions, local)
50
+ - **Latest**: Resolves to V1
51
+
52
+ **Important**: V0 and V1 signatures are **not comparable** due to different embedding spaces.
53
+
54
+ ## Documentation
55
+
56
+ For complete documentation, see:
57
+ - [Python SDK README](README.md)
58
+ - [Algorithm Specification](../../spec/SPEC.md)
59
+ - [Signature Versioning](../../spec/VERSIONING.md)
@@ -0,0 +1,229 @@
1
+ Metadata-Version: 2.5
2
+ Name: 0din-prompt-toolkit
3
+ Version: 0.9.3
4
+ Summary: Multi-language SDK for LSH signature generation for AI prompt similarity detection
5
+ Project-URL: Homepage, https://github.com/0din-ai/prompt-toolkit
6
+ Project-URL: Documentation, https://github.com/0din-ai/prompt-toolkit/tree/main/python
7
+ Project-URL: Repository, https://github.com/0din-ai/prompt-toolkit
8
+ Project-URL: Issues, https://github.com/0din-ai/prompt-toolkit/issues
9
+ Author: 0din Team
10
+ License: Apache-2.0
11
+ Keywords: ai,embeddings,locality-sensitive-hashing,lsh,similarity
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: Apache Software License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Programming Language :: Python :: 3.13
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: numpy>=1.24.0
23
+ Provides-Extra: all
24
+ Requires-Dist: 0din-prompt-toolkit-native>=0.1.0; extra == 'all'
25
+ Requires-Dist: aiohttp>=3.9.0; extra == 'all'
26
+ Requires-Dist: onnxruntime>=1.16.0; extra == 'all'
27
+ Requires-Dist: openai>=1.0.0; extra == 'all'
28
+ Requires-Dist: scikit-learn>=1.3.0; extra == 'all'
29
+ Requires-Dist: sentence-transformers>=2.2.0; extra == 'all'
30
+ Requires-Dist: torch>=2.0.0; extra == 'all'
31
+ Requires-Dist: transformers>=4.0.0; extra == 'all'
32
+ Provides-Extra: cm-lsh
33
+ Requires-Dist: numpy>=1.24.0; extra == 'cm-lsh'
34
+ Requires-Dist: scikit-learn>=1.3.0; extra == 'cm-lsh'
35
+ Provides-Extra: dev
36
+ Requires-Dist: aiohttp>=3.9.0; extra == 'dev'
37
+ Requires-Dist: black>=23.0.0; extra == 'dev'
38
+ Requires-Dist: mypy>=1.5.0; extra == 'dev'
39
+ Requires-Dist: pdoc>=14.0.0; extra == 'dev'
40
+ Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
41
+ Requires-Dist: pytest>=7.4.0; extra == 'dev'
42
+ Requires-Dist: ruff>=0.1.0; extra == 'dev'
43
+ Provides-Extra: native
44
+ Requires-Dist: 0din-prompt-toolkit-native>=0.1.0; extra == 'native'
45
+ Provides-Extra: onnx
46
+ Requires-Dist: onnxruntime>=1.16.0; extra == 'onnx'
47
+ Requires-Dist: sentence-transformers>=2.2.0; extra == 'onnx'
48
+ Requires-Dist: transformers>=4.0.0; extra == 'onnx'
49
+ Provides-Extra: openai
50
+ Requires-Dist: openai>=1.0.0; extra == 'openai'
51
+ Provides-Extra: susfactor
52
+ Requires-Dist: torch>=2.0.0; extra == 'susfactor'
53
+ Requires-Dist: transformers>=4.0.0; extra == 'susfactor'
54
+ Provides-Extra: threatfeed
55
+ Requires-Dist: aiohttp>=3.9.0; extra == 'threatfeed'
56
+ Description-Content-Type: text/markdown
57
+
58
+ # odin-prompt-toolkit (Python)
59
+
60
+ Multi-language SDK for LSH (Locality-Sensitive Hashing) signature generation for AI prompt similarity detection.
61
+
62
+ This is the Python implementation of the odin-prompt-toolkit algorithm, also available in [Rust](../rust) and [TypeScript](../typescript).
63
+
64
+ ## Installation
65
+
66
+ `0din-prompt-toolkit` ships as **two packages** so you only ship compiled code when you want it:
67
+
68
+ | Package | What it is | When you get it |
69
+ |---------|-----------|-----------------|
70
+ | `0din-prompt-toolkit` | Pure-Python core. One universal (`py3-none-any`) wheel — installs on any OS/arch/Python, **no compiler**. | Always (base install) |
71
+ | `0din-prompt-toolkit-native` | Optional Rust accelerator (PyO3). **Prebuilt** wheels for Linux/macOS/Windows × CPython 3.10–3.13. | Only with the `[native]` extra |
72
+
73
+ ```bash
74
+ # Pure Python — works everywhere, slower signature generation
75
+ pip install 0din-prompt-toolkit
76
+
77
+ # With native Rust acceleration — recommended for production
78
+ pip install "0din-prompt-toolkit[native]"
79
+
80
+ # Optional features
81
+ pip install "0din-prompt-toolkit[onnx]" # local ONNX embeddings + SusFactor
82
+ pip install "0din-prompt-toolkit[openai]" # OpenAI embeddings
83
+ pip install "0din-prompt-toolkit[cm-lsh]" # Confidence Matrix LSH
84
+ pip install "0din-prompt-toolkit[threatfeed]" # 0DIN threat feed
85
+ pip install "0din-prompt-toolkit[all]" # everything, including native
86
+ ```
87
+
88
+ From git (development):
89
+
90
+ ```bash
91
+ pip install "0din-prompt-toolkit[native] @ git+https://github.com/0din-ai/prompt-toolkit#subdirectory=packages/python"
92
+ ```
93
+
94
+ ## Native vs pure Python
95
+
96
+ Both paths produce **bit-identical signatures** (verified across all implementations via canonical test vectors). They differ only in **how they install** and **how fast** signature generation runs.
97
+
98
+ - **Pure Python** (`0din-prompt-toolkit`) — a single universal wheel with no compiled code. Installs anywhere with zero build tools. Signature generation runs in a Python loop.
99
+ - **Native** (`[native]` → `0din-prompt-toolkit-native`) — a prebuilt compiled Rust extension. `pip` downloads a wheel matching your platform, so **no Rust toolchain or compiler is required** on Linux/macOS/Windows with CPython 3.10–3.13. On any platform without a matching wheel, the base package still installs and **automatically falls back to pure Python** — nothing breaks, it's just slower.
100
+
101
+ The accelerator is **transparent** — the same API uses native automatically when it's present:
102
+
103
+ ```python
104
+ from odin_prompt_toolkit import NATIVE_AVAILABLE
105
+ print("native" if NATIVE_AVAILABLE else "pure Python")
106
+ ```
107
+
108
+ Force pure Python even when native is installed: `export ODIN_PROMPT_TOOLKIT_NO_NATIVE=1`.
109
+
110
+ ### Speed tradeoff
111
+
112
+ Native replaces the hot signature-generation loop with compiled, SIMD-optimized Rust:
113
+
114
+ | | Native (Rust) | Pure Python |
115
+ |---|---|---|
116
+ | Throughput | ~5,300 sigs/sec | ~85 sigs/sec (384-dim) … ~9 sigs/sec (1024-dim) |
117
+ | Per signature | ~0.2 ms | ~12 ms … ~115 ms |
118
+ | vs native | — | **~60×–600× slower** |
119
+
120
+ The multiplier depends on **embedding dimension**: pure Python loops over every dimension, so the gap widens with larger vectors (~63× at 384-dim, ~590× at 1024-dim). Native throughput is roughly constant.
121
+
122
+ **What it means end-to-end:** in a real pipeline, embedding generation usually dominates. On the 3,714-prompt benchmark (local ONNX, CPU) embedding took 112.6 s; adding signature generation cost **0.7 s with native (+0.6%)** vs **43.8 s pure Python (+38%)**. When embeddings are pre-computed or cached (e.g. real-time dedup), signature generation *is* the cost — and the full speedup applies.
123
+
124
+ **Rule of thumb:** use `[native]` in production; pure Python is a correct, always-available fallback for prototyping or unusual platforms.
125
+
126
+ ## Quick Start
127
+
128
+ ### Basic LSH Signatures
129
+
130
+ ```python
131
+ from odin_prompt_toolkit import simhash_lsh_multi, normalize_vector
132
+
133
+ # Your embedding vector (must be L2-normalized)
134
+ vector = [0.5, 0.5, 0.5, 0.5]
135
+ normalized = normalize_vector(vector)
136
+
137
+ # Generate LSH signatures (3 families, 256 bits, 16 bands)
138
+ families = simhash_lsh_multi(normalized)
139
+
140
+ print(f"Signature: {families[0].signature}")
141
+ print(f"Bands: {families[0].bands}")
142
+ ```
143
+
144
+ ### Similarity Comparison
145
+
146
+ ```python
147
+ from odin_prompt_toolkit import simhash_lsh_multi, hamming_distance_hex, cosine_from_hamming
148
+
149
+ # Generate signatures for two vectors
150
+ families1 = simhash_lsh_multi(vector1)
151
+ families2 = simhash_lsh_multi(vector2)
152
+
153
+ # Compute Hamming distance
154
+ distance = hamming_distance_hex(families1[0].signature, families2[0].signature)
155
+
156
+ # Estimate cosine similarity
157
+ similarity = cosine_from_hamming(distance, 256)
158
+ print(f"Estimated cosine similarity: {similarity:.3f}")
159
+ ```
160
+
161
+ ### Confidence Matrix LSH (CM-LSH)
162
+
163
+ ```python
164
+ from odin_prompt_toolkit.cm_lsh import create_default_cm_lsh
165
+
166
+ # Create CM-LSH hasher (1024 dimensions)
167
+ cm_lsh = create_default_cm_lsh(1024, family=0)
168
+
169
+ # Generate 512-bit signature with confidence matrix
170
+ hash1 = cm_lsh.hash(embedding1)
171
+ hash2 = cm_lsh.hash(embedding2)
172
+
173
+ # Compute calibrated similarity
174
+ similarity = cm_lsh.sim(hash1, hash2)
175
+ print(f"CM-LSH similarity: {similarity:.3f}")
176
+
177
+ # Check for duplicates
178
+ is_duplicate = cm_lsh.is_dup(hash1, hash2, threshold=0.85)
179
+ ```
180
+
181
+ ## Signature Versions
182
+
183
+ - **V0**: OpenAI text-embedding-3-large (1536 dimensions, API-based)
184
+ - **V1**: 0din-jailbreak-embeddings-small ONNX (1024 dimensions, local)
185
+ - **Latest**: Resolves to V1
186
+
187
+ **Important**: V0 and V1 signatures are **not comparable** due to different embedding spaces.
188
+
189
+ ## Algorithm
190
+
191
+ SimHash via Random Hyperplane LSH (Charikar 2002):
192
+ - Deterministic hyperplanes via SplitMix64 PRNG
193
+ - Default: 3 families × 256 bits × 16 bands
194
+ - Hex-encoded signatures (64 hex chars = 256 bits)
195
+ - Hamming distance → cosine similarity via `cos(π × d/n)`
196
+
197
+ See the [specification](../../spec/SPEC.md) for complete algorithm details.
198
+
199
+ ## Development
200
+
201
+ ### Setup
202
+
203
+ ```bash
204
+ cd python
205
+ pip install -e ".[dev]"
206
+ ```
207
+
208
+ ### Run Tests
209
+
210
+ ```bash
211
+ pytest tests/
212
+ ```
213
+
214
+ ### Type Checking
215
+
216
+ ```bash
217
+ mypy odin_prompt_toolkit/
218
+ ```
219
+
220
+ ### Formatting
221
+
222
+ ```bash
223
+ black odin_prompt_toolkit/ tests/
224
+ ruff check odin_prompt_toolkit/ tests/
225
+ ```
226
+
227
+ ## License
228
+
229
+ Apache License 2.0
@@ -0,0 +1,172 @@
1
+ # odin-prompt-toolkit (Python)
2
+
3
+ Multi-language SDK for LSH (Locality-Sensitive Hashing) signature generation for AI prompt similarity detection.
4
+
5
+ This is the Python implementation of the odin-prompt-toolkit algorithm, also available in [Rust](../rust) and [TypeScript](../typescript).
6
+
7
+ ## Installation
8
+
9
+ `0din-prompt-toolkit` ships as **two packages** so you only ship compiled code when you want it:
10
+
11
+ | Package | What it is | When you get it |
12
+ |---------|-----------|-----------------|
13
+ | `0din-prompt-toolkit` | Pure-Python core. One universal (`py3-none-any`) wheel — installs on any OS/arch/Python, **no compiler**. | Always (base install) |
14
+ | `0din-prompt-toolkit-native` | Optional Rust accelerator (PyO3). **Prebuilt** wheels for Linux/macOS/Windows × CPython 3.10–3.13. | Only with the `[native]` extra |
15
+
16
+ ```bash
17
+ # Pure Python — works everywhere, slower signature generation
18
+ pip install 0din-prompt-toolkit
19
+
20
+ # With native Rust acceleration — recommended for production
21
+ pip install "0din-prompt-toolkit[native]"
22
+
23
+ # Optional features
24
+ pip install "0din-prompt-toolkit[onnx]" # local ONNX embeddings + SusFactor
25
+ pip install "0din-prompt-toolkit[openai]" # OpenAI embeddings
26
+ pip install "0din-prompt-toolkit[cm-lsh]" # Confidence Matrix LSH
27
+ pip install "0din-prompt-toolkit[threatfeed]" # 0DIN threat feed
28
+ pip install "0din-prompt-toolkit[all]" # everything, including native
29
+ ```
30
+
31
+ From git (development):
32
+
33
+ ```bash
34
+ pip install "0din-prompt-toolkit[native] @ git+https://github.com/0din-ai/prompt-toolkit#subdirectory=packages/python"
35
+ ```
36
+
37
+ ## Native vs pure Python
38
+
39
+ Both paths produce **bit-identical signatures** (verified across all implementations via canonical test vectors). They differ only in **how they install** and **how fast** signature generation runs.
40
+
41
+ - **Pure Python** (`0din-prompt-toolkit`) — a single universal wheel with no compiled code. Installs anywhere with zero build tools. Signature generation runs in a Python loop.
42
+ - **Native** (`[native]` → `0din-prompt-toolkit-native`) — a prebuilt compiled Rust extension. `pip` downloads a wheel matching your platform, so **no Rust toolchain or compiler is required** on Linux/macOS/Windows with CPython 3.10–3.13. On any platform without a matching wheel, the base package still installs and **automatically falls back to pure Python** — nothing breaks, it's just slower.
43
+
44
+ The accelerator is **transparent** — the same API uses native automatically when it's present:
45
+
46
+ ```python
47
+ from odin_prompt_toolkit import NATIVE_AVAILABLE
48
+ print("native" if NATIVE_AVAILABLE else "pure Python")
49
+ ```
50
+
51
+ Force pure Python even when native is installed: `export ODIN_PROMPT_TOOLKIT_NO_NATIVE=1`.
52
+
53
+ ### Speed tradeoff
54
+
55
+ Native replaces the hot signature-generation loop with compiled, SIMD-optimized Rust:
56
+
57
+ | | Native (Rust) | Pure Python |
58
+ |---|---|---|
59
+ | Throughput | ~5,300 sigs/sec | ~85 sigs/sec (384-dim) … ~9 sigs/sec (1024-dim) |
60
+ | Per signature | ~0.2 ms | ~12 ms … ~115 ms |
61
+ | vs native | — | **~60×–600× slower** |
62
+
63
+ The multiplier depends on **embedding dimension**: pure Python loops over every dimension, so the gap widens with larger vectors (~63× at 384-dim, ~590× at 1024-dim). Native throughput is roughly constant.
64
+
65
+ **What it means end-to-end:** in a real pipeline, embedding generation usually dominates. On the 3,714-prompt benchmark (local ONNX, CPU) embedding took 112.6 s; adding signature generation cost **0.7 s with native (+0.6%)** vs **43.8 s pure Python (+38%)**. When embeddings are pre-computed or cached (e.g. real-time dedup), signature generation *is* the cost — and the full speedup applies.
66
+
67
+ **Rule of thumb:** use `[native]` in production; pure Python is a correct, always-available fallback for prototyping or unusual platforms.
68
+
69
+ ## Quick Start
70
+
71
+ ### Basic LSH Signatures
72
+
73
+ ```python
74
+ from odin_prompt_toolkit import simhash_lsh_multi, normalize_vector
75
+
76
+ # Your embedding vector (must be L2-normalized)
77
+ vector = [0.5, 0.5, 0.5, 0.5]
78
+ normalized = normalize_vector(vector)
79
+
80
+ # Generate LSH signatures (3 families, 256 bits, 16 bands)
81
+ families = simhash_lsh_multi(normalized)
82
+
83
+ print(f"Signature: {families[0].signature}")
84
+ print(f"Bands: {families[0].bands}")
85
+ ```
86
+
87
+ ### Similarity Comparison
88
+
89
+ ```python
90
+ from odin_prompt_toolkit import simhash_lsh_multi, hamming_distance_hex, cosine_from_hamming
91
+
92
+ # Generate signatures for two vectors
93
+ families1 = simhash_lsh_multi(vector1)
94
+ families2 = simhash_lsh_multi(vector2)
95
+
96
+ # Compute Hamming distance
97
+ distance = hamming_distance_hex(families1[0].signature, families2[0].signature)
98
+
99
+ # Estimate cosine similarity
100
+ similarity = cosine_from_hamming(distance, 256)
101
+ print(f"Estimated cosine similarity: {similarity:.3f}")
102
+ ```
103
+
104
+ ### Confidence Matrix LSH (CM-LSH)
105
+
106
+ ```python
107
+ from odin_prompt_toolkit.cm_lsh import create_default_cm_lsh
108
+
109
+ # Create CM-LSH hasher (1024 dimensions)
110
+ cm_lsh = create_default_cm_lsh(1024, family=0)
111
+
112
+ # Generate 512-bit signature with confidence matrix
113
+ hash1 = cm_lsh.hash(embedding1)
114
+ hash2 = cm_lsh.hash(embedding2)
115
+
116
+ # Compute calibrated similarity
117
+ similarity = cm_lsh.sim(hash1, hash2)
118
+ print(f"CM-LSH similarity: {similarity:.3f}")
119
+
120
+ # Check for duplicates
121
+ is_duplicate = cm_lsh.is_dup(hash1, hash2, threshold=0.85)
122
+ ```
123
+
124
+ ## Signature Versions
125
+
126
+ - **V0**: OpenAI text-embedding-3-large (1536 dimensions, API-based)
127
+ - **V1**: 0din-jailbreak-embeddings-small ONNX (1024 dimensions, local)
128
+ - **Latest**: Resolves to V1
129
+
130
+ **Important**: V0 and V1 signatures are **not comparable** due to different embedding spaces.
131
+
132
+ ## Algorithm
133
+
134
+ SimHash via Random Hyperplane LSH (Charikar 2002):
135
+ - Deterministic hyperplanes via SplitMix64 PRNG
136
+ - Default: 3 families × 256 bits × 16 bands
137
+ - Hex-encoded signatures (64 hex chars = 256 bits)
138
+ - Hamming distance → cosine similarity via `cos(π × d/n)`
139
+
140
+ See the [specification](../../spec/SPEC.md) for complete algorithm details.
141
+
142
+ ## Development
143
+
144
+ ### Setup
145
+
146
+ ```bash
147
+ cd python
148
+ pip install -e ".[dev]"
149
+ ```
150
+
151
+ ### Run Tests
152
+
153
+ ```bash
154
+ pytest tests/
155
+ ```
156
+
157
+ ### Type Checking
158
+
159
+ ```bash
160
+ mypy odin_prompt_toolkit/
161
+ ```
162
+
163
+ ### Formatting
164
+
165
+ ```bash
166
+ black odin_prompt_toolkit/ tests/
167
+ ruff check odin_prompt_toolkit/ tests/
168
+ ```
169
+
170
+ ## License
171
+
172
+ Apache License 2.0
@@ -0,0 +1,63 @@
1
+ #!/usr/bin/env python3
2
+ """Basic LSH signature generation example.
3
+
4
+ This example demonstrates:
5
+ - Generating an LSH signature from a normalized vector
6
+ - Default configuration (3 families, 256 bits, 16 bands)
7
+ - Formatting and parsing signature strings
8
+
9
+ Run with: python python/examples/basic_signature.py
10
+ """
11
+
12
+ from odin_prompt_toolkit import simhash_lsh_multi
13
+
14
+
15
+ def main():
16
+ print("=== Basic LSH Signature Generation ===\n")
17
+
18
+ # Example normalized vector (4 dimensions for clarity)
19
+ # In practice, this would come from an embedding model (384 or 1536 dims)
20
+ normalized_vector = [0.5, 0.5, 0.5, 0.5]
21
+
22
+ print(f"Input vector: {normalized_vector}")
23
+ print(f"Vector dimensions: {len(normalized_vector)}\n")
24
+
25
+ # Generate LSH signatures with default configuration
26
+ families_count = 3
27
+ bits = 256
28
+ bands = 16
29
+
30
+ print("Configuration:")
31
+ print(f" Families: {families_count}")
32
+ print(f" Bits per signature: {bits}")
33
+ print(f" Bands: {bands}\n")
34
+
35
+ families = simhash_lsh_multi(normalized_vector, families=families_count, bits=bits, bands=bands)
36
+
37
+ # Display results for each family
38
+ for family in families:
39
+ print(f"Family {family.family}:")
40
+ print(f" Signature (hex): {family.signature}")
41
+ print(f" Signature length: {len(family.signature)} hex chars = {family.bits} bits")
42
+ print(f" Number of bands: {len(family.bands)}")
43
+ print(f" Band 0: {family.bands[0]} (first {len(family.bands[0])} hex chars)")
44
+ print()
45
+
46
+ # Format as 0din signature string (V1 format)
47
+ primary_sig = families[0].signature
48
+ signature_string = f"0din-v1:{primary_sig}"
49
+
50
+ print("Formatted signature string:")
51
+ print(f" {signature_string}")
52
+ print()
53
+
54
+ # In a real application, you would:
55
+ # 1. Store this signature in a database with the original text
56
+ # 2. Use bands for efficient similarity search (LSH indexing)
57
+ # 3. Compare signatures using hamming distance
58
+
59
+ print("✓ Signature generation complete!")
60
+
61
+
62
+ if __name__ == "__main__":
63
+ main()
@@ -0,0 +1,88 @@
1
+ #!/usr/bin/env python3
2
+ """Confidence Matrix LSH (CM-LSH) example.
3
+
4
+ This example demonstrates:
5
+ - Enhanced LSH with confidence matrix
6
+ - Dual hash structure (512-bit signature + 512-bit confidence)
7
+ - Backward compatibility with standard LSH (first 256 bits)
8
+ - Calibrated similarity estimation
9
+
10
+ Run with: python python/examples/cm_lsh_example.py
11
+ """
12
+
13
+ try:
14
+ from odin_prompt_toolkit.cm_lsh import create_default_cm_lsh
15
+ from odin_prompt_toolkit import normalize_vector
16
+ except ImportError:
17
+ print("This example requires the CM-LSH implementation.")
18
+ print("Install with: pip install -e '.[cm-lsh]'")
19
+ exit(1)
20
+
21
+
22
+ def main():
23
+ print("=== Confidence Matrix LSH (CM-LSH) ===\n")
24
+
25
+ # Example vectors
26
+ vector_a = [1.0, 1.0, 1.0, 1.0]
27
+ vector_b = [1.0, 0.9, 1.1, 1.0] # Similar to A
28
+ vector_c = [-1.0, -1.0, -1.0, -1.0] # Opposite to A
29
+
30
+ print("Input vectors:")
31
+ print(f" A: {vector_a}")
32
+ print(f" B: {vector_b} (similar to A)")
33
+ print(f" C: {vector_c} (opposite to A)\n")
34
+
35
+ # Normalize vectors
36
+ norm_a = normalize_vector(vector_a)
37
+ norm_b = normalize_vector(vector_b)
38
+ norm_c = normalize_vector(vector_c)
39
+
40
+ # Create CM-LSH hasher with default configuration
41
+ # This uses identity ITQ (no learned rotation) for simplicity
42
+ # Family 0 for deterministic results
43
+ hasher = create_default_cm_lsh(len(norm_a), family=0)
44
+
45
+ print("CM-LSH Configuration:")
46
+ print(" Total bits: 512 (256 LSH-TS + 256 ITQ)")
47
+ print(" First 256 bits: LSH-TS compatible")
48
+ print(" Confidence matrix: Alpha-weighted agreement\n")
49
+
50
+ # Generate dual hashes
51
+ hash_a = hasher.hash(norm_a)
52
+ hash_b = hasher.hash(norm_b)
53
+ hash_c = hasher.hash(norm_c)
54
+
55
+ print("Dual hashes (showing first 32 hex chars of 128):")
56
+ print(f" A: hash={hash_a.hash_a[:32]} conf={hash_a.hash_b[:32]}")
57
+ print(f" B: hash={hash_b.hash_a[:32]} conf={hash_b.hash_b[:32]}")
58
+ print(f" C: hash={hash_c.hash_a[:32]} conf={hash_c.hash_b[:32]}")
59
+ print()
60
+
61
+ # Demonstrate LSH-TS backward compatibility
62
+ print("LSH-TS compatibility (first 256 bits):")
63
+ print(f" A: {hash_a.lsh_ts_compat()[:16]}")
64
+ print(f" B: {hash_b.lsh_ts_compat()[:16]}")
65
+ print(f" C: {hash_c.lsh_ts_compat()[:16]}")
66
+ print(" (showing first 16 hex chars of 64)\n")
67
+
68
+ # Compute calibrated similarities
69
+ print("Calibrated similarities:\n")
70
+
71
+ sim_ab = hasher.sim(hash_a, hash_b)
72
+ sim_ac = hasher.sim(hash_a, hash_c)
73
+ sim_bc = hasher.sim(hash_b, hash_c)
74
+
75
+ print(f" A vs B: {sim_ab:.4f}")
76
+ print(f" A vs C: {sim_ac:.4f}")
77
+ print(f" B vs C: {sim_bc:.4f}")
78
+
79
+ print("\n✓ CM-LSH example complete!")
80
+ print("\nKey advantages of CM-LSH:")
81
+ print(" - Confidence matrix weights reliable bits higher")
82
+ print(" - Isotonic calibration improves similarity estimates")
83
+ print(" - Dual hash (LSH + ITQ) for better quantization")
84
+ print(" - Backward compatible with standard LSH")
85
+
86
+
87
+ if __name__ == "__main__":
88
+ main()