0din-prompt-toolkit 0.9.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- 0din_prompt_toolkit-0.9.3/.gitignore +69 -0
- 0din_prompt_toolkit-0.9.3/INSTALL.md +59 -0
- 0din_prompt_toolkit-0.9.3/PKG-INFO +229 -0
- 0din_prompt_toolkit-0.9.3/README.md +172 -0
- 0din_prompt_toolkit-0.9.3/examples/basic_signature.py +63 -0
- 0din_prompt_toolkit-0.9.3/examples/cm_lsh_example.py +88 -0
- 0din_prompt_toolkit-0.9.3/examples/duplicate_detection.py +94 -0
- 0din_prompt_toolkit-0.9.3/examples/sign_text.py +158 -0
- 0din_prompt_toolkit-0.9.3/examples/similarity_comparison.py +75 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/__init__.py +126 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/_accel.py +93 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/cm_lsh.py +463 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/error.py +116 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/hasher.py +46 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/hashers/__init__.py +31 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/hashers/lsh.py +56 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/lsh.py +223 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/provider.py +80 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/__init__.py +27 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/model_cache.py +219 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/onnx.py +220 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/providers/openai.py +126 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/py.typed +0 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/sign.py +233 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/__init__.py +74 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/classifier.py +353 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/compare.py +72 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/onnx_classifier.py +378 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/susfactor/types.py +143 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/__init__.py +38 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/cache.py +325 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/client.py +197 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/compare.py +43 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/threatfeed/types.py +112 -0
- 0din_prompt_toolkit-0.9.3/odin_prompt_toolkit/types.py +276 -0
- 0din_prompt_toolkit-0.9.3/pyproject.toml +166 -0
- 0din_prompt_toolkit-0.9.3/tests/test_cm_lsh_vectors.py +118 -0
- 0din_prompt_toolkit-0.9.3/tests/test_cross_validation.py +148 -0
- 0din_prompt_toolkit-0.9.3/tests/test_error.py +126 -0
- 0din_prompt_toolkit-0.9.3/tests/test_hasher.py +63 -0
- 0din_prompt_toolkit-0.9.3/tests/test_sign_text.py +172 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_cache.py +97 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_chunking.py +251 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_classifier.py +225 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_compare.py +69 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_integration.py +47 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_onnx_parity.py +398 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_parity.py +142 -0
- 0din_prompt_toolkit-0.9.3/tests/test_susfactor_types.py +145 -0
- 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_cache.py +177 -0
- 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_client.py +396 -0
- 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_compare.py +110 -0
- 0din_prompt_toolkit-0.9.3/tests/test_threatfeed_fixture.py +136 -0
- 0din_prompt_toolkit-0.9.3/tests/test_tokenizer_parity.py +79 -0
- 0din_prompt_toolkit-0.9.3/tests/test_types.py +88 -0
- 0din_prompt_toolkit-0.9.3/tests/test_vectors.py +179 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# Build artifacts
|
|
2
|
+
/dist/
|
|
3
|
+
|
|
4
|
+
# Rust
|
|
5
|
+
/packages/rust/target/
|
|
6
|
+
/packages/python-native/target/
|
|
7
|
+
*.rs.bk
|
|
8
|
+
|
|
9
|
+
# Python
|
|
10
|
+
/packages/python/__pycache__/
|
|
11
|
+
/packages/python/*.py[cod]
|
|
12
|
+
/packages/python/*$py.class
|
|
13
|
+
/packages/python/.pytest_cache/
|
|
14
|
+
/packages/python/.coverage
|
|
15
|
+
/packages/python/htmlcov/
|
|
16
|
+
/packages/python/dist/
|
|
17
|
+
/packages/python/build/
|
|
18
|
+
/packages/python/*.egg-info/
|
|
19
|
+
/packages/python/.mypy_cache/
|
|
20
|
+
/packages/python/.ruff_cache/
|
|
21
|
+
/packages/python/.hatch/
|
|
22
|
+
|
|
23
|
+
# TypeScript
|
|
24
|
+
/packages/typescript/node_modules/
|
|
25
|
+
/packages/typescript/dist/
|
|
26
|
+
/packages/typescript/*.tgz
|
|
27
|
+
/packages/typescript/.turbo/
|
|
28
|
+
|
|
29
|
+
# Documentation
|
|
30
|
+
/docs/node_modules/
|
|
31
|
+
/docs/.docusaurus/
|
|
32
|
+
/docs/build/
|
|
33
|
+
|
|
34
|
+
# IDE
|
|
35
|
+
.vscode/
|
|
36
|
+
.idea/
|
|
37
|
+
*.swp
|
|
38
|
+
*.swo
|
|
39
|
+
*~
|
|
40
|
+
|
|
41
|
+
# OS
|
|
42
|
+
.DS_Store
|
|
43
|
+
Thumbs.db
|
|
44
|
+
|
|
45
|
+
# Temporary files
|
|
46
|
+
*.log
|
|
47
|
+
*.tmp
|
|
48
|
+
.env
|
|
49
|
+
.env.local
|
|
50
|
+
demos/cache/*.npz
|
|
51
|
+
demos/cache/*.json
|
|
52
|
+
demos/cache/*.db
|
|
53
|
+
|
|
54
|
+
# OpenCode
|
|
55
|
+
.opencode/
|
|
56
|
+
|
|
57
|
+
# Model files (large ONNX models, don't commit to repo)
|
|
58
|
+
models/*/onnx/*.onnx
|
|
59
|
+
models/*/onnx/*.onnx_data
|
|
60
|
+
models/*/tokenizer.json
|
|
61
|
+
models/*/*.json
|
|
62
|
+
# ...but keep the small metadata config.json (tracked for every model version)
|
|
63
|
+
!models/*/config.json
|
|
64
|
+
docs/static/api/rust/
|
|
65
|
+
|
|
66
|
+
# build/test artifacts (accidentally committed once; keep untracked)
|
|
67
|
+
packages/typescript/coverage/
|
|
68
|
+
*.db-shm
|
|
69
|
+
*.db-wal
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# Installing odin-prompt-toolkit (Python)
|
|
2
|
+
|
|
3
|
+
## Requirements
|
|
4
|
+
- Python >= 3.10
|
|
5
|
+
|
|
6
|
+
## Install from wheel
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install odin_prompt_toolkit-<VERSION>-py3-none-any.whl
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
Replace `<VERSION>` with the version number (e.g., `0.1.1`).
|
|
13
|
+
|
|
14
|
+
## Install with optional features
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
# OpenAI embeddings (API-based)
|
|
18
|
+
pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[openai]"
|
|
19
|
+
|
|
20
|
+
# ONNX embeddings (local, no API key needed)
|
|
21
|
+
pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[onnx]"
|
|
22
|
+
|
|
23
|
+
# Confidence Matrix LSH (higher accuracy)
|
|
24
|
+
pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[cm-lsh]"
|
|
25
|
+
|
|
26
|
+
# All features
|
|
27
|
+
pip install "odin_prompt_toolkit-<VERSION>-py3-none-any.whl[all]"
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Quick Start
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from odin_prompt_toolkit import simhash_lsh_multi, normalize_vector
|
|
34
|
+
|
|
35
|
+
# Your embedding vector (must be L2-normalized)
|
|
36
|
+
vector = [0.5, 0.5, 0.5, 0.5]
|
|
37
|
+
normalized = normalize_vector(vector)
|
|
38
|
+
|
|
39
|
+
# Generate LSH signatures (3 families, 256 bits, 16 bands)
|
|
40
|
+
families = simhash_lsh_multi(normalized)
|
|
41
|
+
|
|
42
|
+
print(f"Signature: {families[0].signature}")
|
|
43
|
+
print(f"Bands: {families[0].bands}")
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
## Signature Versions
|
|
47
|
+
|
|
48
|
+
- **V0**: OpenAI text-embedding-3-large (1536 dimensions, API-based)
|
|
49
|
+
- **V1**: 0din-jailbreak-embeddings-small ONNX (1024 dimensions, local)
|
|
50
|
+
- **Latest**: Resolves to V1
|
|
51
|
+
|
|
52
|
+
**Important**: V0 and V1 signatures are **not comparable** due to different embedding spaces.
|
|
53
|
+
|
|
54
|
+
## Documentation
|
|
55
|
+
|
|
56
|
+
For complete documentation, see:
|
|
57
|
+
- [Python SDK README](README.md)
|
|
58
|
+
- [Algorithm Specification](../../spec/SPEC.md)
|
|
59
|
+
- [Signature Versioning](../../spec/VERSIONING.md)
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: 0din-prompt-toolkit
|
|
3
|
+
Version: 0.9.3
|
|
4
|
+
Summary: Multi-language SDK for LSH signature generation for AI prompt similarity detection
|
|
5
|
+
Project-URL: Homepage, https://github.com/0din-ai/prompt-toolkit
|
|
6
|
+
Project-URL: Documentation, https://github.com/0din-ai/prompt-toolkit/tree/main/python
|
|
7
|
+
Project-URL: Repository, https://github.com/0din-ai/prompt-toolkit
|
|
8
|
+
Project-URL: Issues, https://github.com/0din-ai/prompt-toolkit/issues
|
|
9
|
+
Author: 0din Team
|
|
10
|
+
License: Apache-2.0
|
|
11
|
+
Keywords: ai,embeddings,locality-sensitive-hashing,lsh,similarity
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: numpy>=1.24.0
|
|
23
|
+
Provides-Extra: all
|
|
24
|
+
Requires-Dist: 0din-prompt-toolkit-native>=0.1.0; extra == 'all'
|
|
25
|
+
Requires-Dist: aiohttp>=3.9.0; extra == 'all'
|
|
26
|
+
Requires-Dist: onnxruntime>=1.16.0; extra == 'all'
|
|
27
|
+
Requires-Dist: openai>=1.0.0; extra == 'all'
|
|
28
|
+
Requires-Dist: scikit-learn>=1.3.0; extra == 'all'
|
|
29
|
+
Requires-Dist: sentence-transformers>=2.2.0; extra == 'all'
|
|
30
|
+
Requires-Dist: torch>=2.0.0; extra == 'all'
|
|
31
|
+
Requires-Dist: transformers>=4.0.0; extra == 'all'
|
|
32
|
+
Provides-Extra: cm-lsh
|
|
33
|
+
Requires-Dist: numpy>=1.24.0; extra == 'cm-lsh'
|
|
34
|
+
Requires-Dist: scikit-learn>=1.3.0; extra == 'cm-lsh'
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: aiohttp>=3.9.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: black>=23.0.0; extra == 'dev'
|
|
38
|
+
Requires-Dist: mypy>=1.5.0; extra == 'dev'
|
|
39
|
+
Requires-Dist: pdoc>=14.0.0; extra == 'dev'
|
|
40
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
|
|
41
|
+
Requires-Dist: pytest>=7.4.0; extra == 'dev'
|
|
42
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
43
|
+
Provides-Extra: native
|
|
44
|
+
Requires-Dist: 0din-prompt-toolkit-native>=0.1.0; extra == 'native'
|
|
45
|
+
Provides-Extra: onnx
|
|
46
|
+
Requires-Dist: onnxruntime>=1.16.0; extra == 'onnx'
|
|
47
|
+
Requires-Dist: sentence-transformers>=2.2.0; extra == 'onnx'
|
|
48
|
+
Requires-Dist: transformers>=4.0.0; extra == 'onnx'
|
|
49
|
+
Provides-Extra: openai
|
|
50
|
+
Requires-Dist: openai>=1.0.0; extra == 'openai'
|
|
51
|
+
Provides-Extra: susfactor
|
|
52
|
+
Requires-Dist: torch>=2.0.0; extra == 'susfactor'
|
|
53
|
+
Requires-Dist: transformers>=4.0.0; extra == 'susfactor'
|
|
54
|
+
Provides-Extra: threatfeed
|
|
55
|
+
Requires-Dist: aiohttp>=3.9.0; extra == 'threatfeed'
|
|
56
|
+
Description-Content-Type: text/markdown
|
|
57
|
+
|
|
58
|
+
# odin-prompt-toolkit (Python)
|
|
59
|
+
|
|
60
|
+
Multi-language SDK for LSH (Locality-Sensitive Hashing) signature generation for AI prompt similarity detection.
|
|
61
|
+
|
|
62
|
+
This is the Python implementation of the odin-prompt-toolkit algorithm, also available in [Rust](../rust) and [TypeScript](../typescript).
|
|
63
|
+
|
|
64
|
+
## Installation
|
|
65
|
+
|
|
66
|
+
`0din-prompt-toolkit` ships as **two packages** so you only ship compiled code when you want it:
|
|
67
|
+
|
|
68
|
+
| Package | What it is | When you get it |
|
|
69
|
+
|---------|-----------|-----------------|
|
|
70
|
+
| `0din-prompt-toolkit` | Pure-Python core. One universal (`py3-none-any`) wheel — installs on any OS/arch/Python, **no compiler**. | Always (base install) |
|
|
71
|
+
| `0din-prompt-toolkit-native` | Optional Rust accelerator (PyO3). **Prebuilt** wheels for Linux/macOS/Windows × CPython 3.10–3.13. | Only with the `[native]` extra |
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Pure Python — works everywhere, slower signature generation
|
|
75
|
+
pip install 0din-prompt-toolkit
|
|
76
|
+
|
|
77
|
+
# With native Rust acceleration — recommended for production
|
|
78
|
+
pip install "0din-prompt-toolkit[native]"
|
|
79
|
+
|
|
80
|
+
# Optional features
|
|
81
|
+
pip install "0din-prompt-toolkit[onnx]" # local ONNX embeddings + SusFactor
|
|
82
|
+
pip install "0din-prompt-toolkit[openai]" # OpenAI embeddings
|
|
83
|
+
pip install "0din-prompt-toolkit[cm-lsh]" # Confidence Matrix LSH
|
|
84
|
+
pip install "0din-prompt-toolkit[threatfeed]" # 0DIN threat feed
|
|
85
|
+
pip install "0din-prompt-toolkit[all]" # everything, including native
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
From git (development):
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install "0din-prompt-toolkit[native] @ git+https://github.com/0din-ai/prompt-toolkit#subdirectory=packages/python"
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Native vs pure Python
|
|
95
|
+
|
|
96
|
+
Both paths produce **bit-identical signatures** (verified across all implementations via canonical test vectors). They differ only in **how they install** and **how fast** signature generation runs.
|
|
97
|
+
|
|
98
|
+
- **Pure Python** (`0din-prompt-toolkit`) — a single universal wheel with no compiled code. Installs anywhere with zero build tools. Signature generation runs in a Python loop.
|
|
99
|
+
- **Native** (`[native]` → `0din-prompt-toolkit-native`) — a prebuilt compiled Rust extension. `pip` downloads a wheel matching your platform, so **no Rust toolchain or compiler is required** on Linux/macOS/Windows with CPython 3.10–3.13. On any platform without a matching wheel, the base package still installs and **automatically falls back to pure Python** — nothing breaks, it's just slower.
|
|
100
|
+
|
|
101
|
+
The accelerator is **transparent** — the same API uses native automatically when it's present:
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from odin_prompt_toolkit import NATIVE_AVAILABLE
|
|
105
|
+
print("native" if NATIVE_AVAILABLE else "pure Python")
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Force pure Python even when native is installed: `export ODIN_PROMPT_TOOLKIT_NO_NATIVE=1`.
|
|
109
|
+
|
|
110
|
+
### Speed tradeoff
|
|
111
|
+
|
|
112
|
+
Native replaces the hot signature-generation loop with compiled, SIMD-optimized Rust:
|
|
113
|
+
|
|
114
|
+
| | Native (Rust) | Pure Python |
|
|
115
|
+
|---|---|---|
|
|
116
|
+
| Throughput | ~5,300 sigs/sec | ~85 sigs/sec (384-dim) … ~9 sigs/sec (1024-dim) |
|
|
117
|
+
| Per signature | ~0.2 ms | ~12 ms … ~115 ms |
|
|
118
|
+
| vs native | — | **~60×–600× slower** |
|
|
119
|
+
|
|
120
|
+
The multiplier depends on **embedding dimension**: pure Python loops over every dimension, so the gap widens with larger vectors (~63× at 384-dim, ~590× at 1024-dim). Native throughput is roughly constant.
|
|
121
|
+
|
|
122
|
+
**What it means end-to-end:** in a real pipeline, embedding generation usually dominates. On the 3,714-prompt benchmark (local ONNX, CPU) embedding took 112.6 s; adding signature generation cost **0.7 s with native (+0.6%)** vs **43.8 s pure Python (+38%)**. When embeddings are pre-computed or cached (e.g. real-time dedup), signature generation *is* the cost — and the full speedup applies.
|
|
123
|
+
|
|
124
|
+
**Rule of thumb:** use `[native]` in production; pure Python is a correct, always-available fallback for prototyping or unusual platforms.
|
|
125
|
+
|
|
126
|
+
## Quick Start
|
|
127
|
+
|
|
128
|
+
### Basic LSH Signatures
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
from odin_prompt_toolkit import simhash_lsh_multi, normalize_vector
|
|
132
|
+
|
|
133
|
+
# Your embedding vector (must be L2-normalized)
|
|
134
|
+
vector = [0.5, 0.5, 0.5, 0.5]
|
|
135
|
+
normalized = normalize_vector(vector)
|
|
136
|
+
|
|
137
|
+
# Generate LSH signatures (3 families, 256 bits, 16 bands)
|
|
138
|
+
families = simhash_lsh_multi(normalized)
|
|
139
|
+
|
|
140
|
+
print(f"Signature: {families[0].signature}")
|
|
141
|
+
print(f"Bands: {families[0].bands}")
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### Similarity Comparison
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from odin_prompt_toolkit import simhash_lsh_multi, hamming_distance_hex, cosine_from_hamming
|
|
148
|
+
|
|
149
|
+
# Generate signatures for two vectors
|
|
150
|
+
families1 = simhash_lsh_multi(vector1)
|
|
151
|
+
families2 = simhash_lsh_multi(vector2)
|
|
152
|
+
|
|
153
|
+
# Compute Hamming distance
|
|
154
|
+
distance = hamming_distance_hex(families1[0].signature, families2[0].signature)
|
|
155
|
+
|
|
156
|
+
# Estimate cosine similarity
|
|
157
|
+
similarity = cosine_from_hamming(distance, 256)
|
|
158
|
+
print(f"Estimated cosine similarity: {similarity:.3f}")
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### Confidence Matrix LSH (CM-LSH)
|
|
162
|
+
|
|
163
|
+
```python
|
|
164
|
+
from odin_prompt_toolkit.cm_lsh import create_default_cm_lsh
|
|
165
|
+
|
|
166
|
+
# Create CM-LSH hasher (1024 dimensions)
|
|
167
|
+
cm_lsh = create_default_cm_lsh(1024, family=0)
|
|
168
|
+
|
|
169
|
+
# Generate 512-bit signature with confidence matrix
|
|
170
|
+
hash1 = cm_lsh.hash(embedding1)
|
|
171
|
+
hash2 = cm_lsh.hash(embedding2)
|
|
172
|
+
|
|
173
|
+
# Compute calibrated similarity
|
|
174
|
+
similarity = cm_lsh.sim(hash1, hash2)
|
|
175
|
+
print(f"CM-LSH similarity: {similarity:.3f}")
|
|
176
|
+
|
|
177
|
+
# Check for duplicates
|
|
178
|
+
is_duplicate = cm_lsh.is_dup(hash1, hash2, threshold=0.85)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## Signature Versions
|
|
182
|
+
|
|
183
|
+
- **V0**: OpenAI text-embedding-3-large (1536 dimensions, API-based)
|
|
184
|
+
- **V1**: 0din-jailbreak-embeddings-small ONNX (1024 dimensions, local)
|
|
185
|
+
- **Latest**: Resolves to V1
|
|
186
|
+
|
|
187
|
+
**Important**: V0 and V1 signatures are **not comparable** due to different embedding spaces.
|
|
188
|
+
|
|
189
|
+
## Algorithm
|
|
190
|
+
|
|
191
|
+
SimHash via Random Hyperplane LSH (Charikar 2002):
|
|
192
|
+
- Deterministic hyperplanes via SplitMix64 PRNG
|
|
193
|
+
- Default: 3 families × 256 bits × 16 bands
|
|
194
|
+
- Hex-encoded signatures (64 hex chars = 256 bits)
|
|
195
|
+
- Hamming distance → cosine similarity via `cos(π × d/n)`
|
|
196
|
+
|
|
197
|
+
See the [specification](../../spec/SPEC.md) for complete algorithm details.
|
|
198
|
+
|
|
199
|
+
## Development
|
|
200
|
+
|
|
201
|
+
### Setup
|
|
202
|
+
|
|
203
|
+
```bash
|
|
204
|
+
cd python
|
|
205
|
+
pip install -e ".[dev]"
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
### Run Tests
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
pytest tests/
|
|
212
|
+
```
|
|
213
|
+
|
|
214
|
+
### Type Checking
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
mypy odin_prompt_toolkit/
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
### Formatting
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
black odin_prompt_toolkit/ tests/
|
|
224
|
+
ruff check odin_prompt_toolkit/ tests/
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## License
|
|
228
|
+
|
|
229
|
+
Apache License 2.0
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
# odin-prompt-toolkit (Python)
|
|
2
|
+
|
|
3
|
+
Multi-language SDK for LSH (Locality-Sensitive Hashing) signature generation for AI prompt similarity detection.
|
|
4
|
+
|
|
5
|
+
This is the Python implementation of the odin-prompt-toolkit algorithm, also available in [Rust](../rust) and [TypeScript](../typescript).
|
|
6
|
+
|
|
7
|
+
## Installation
|
|
8
|
+
|
|
9
|
+
`0din-prompt-toolkit` ships as **two packages** so you only ship compiled code when you want it:
|
|
10
|
+
|
|
11
|
+
| Package | What it is | When you get it |
|
|
12
|
+
|---------|-----------|-----------------|
|
|
13
|
+
| `0din-prompt-toolkit` | Pure-Python core. One universal (`py3-none-any`) wheel — installs on any OS/arch/Python, **no compiler**. | Always (base install) |
|
|
14
|
+
| `0din-prompt-toolkit-native` | Optional Rust accelerator (PyO3). **Prebuilt** wheels for Linux/macOS/Windows × CPython 3.10–3.13. | Only with the `[native]` extra |
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
# Pure Python — works everywhere, slower signature generation
|
|
18
|
+
pip install 0din-prompt-toolkit
|
|
19
|
+
|
|
20
|
+
# With native Rust acceleration — recommended for production
|
|
21
|
+
pip install "0din-prompt-toolkit[native]"
|
|
22
|
+
|
|
23
|
+
# Optional features
|
|
24
|
+
pip install "0din-prompt-toolkit[onnx]" # local ONNX embeddings + SusFactor
|
|
25
|
+
pip install "0din-prompt-toolkit[openai]" # OpenAI embeddings
|
|
26
|
+
pip install "0din-prompt-toolkit[cm-lsh]" # Confidence Matrix LSH
|
|
27
|
+
pip install "0din-prompt-toolkit[threatfeed]" # 0DIN threat feed
|
|
28
|
+
pip install "0din-prompt-toolkit[all]" # everything, including native
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
From git (development):
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
pip install "0din-prompt-toolkit[native] @ git+https://github.com/0din-ai/prompt-toolkit#subdirectory=packages/python"
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Native vs pure Python
|
|
38
|
+
|
|
39
|
+
Both paths produce **bit-identical signatures** (verified across all implementations via canonical test vectors). They differ only in **how they install** and **how fast** signature generation runs.
|
|
40
|
+
|
|
41
|
+
- **Pure Python** (`0din-prompt-toolkit`) — a single universal wheel with no compiled code. Installs anywhere with zero build tools. Signature generation runs in a Python loop.
|
|
42
|
+
- **Native** (`[native]` → `0din-prompt-toolkit-native`) — a prebuilt compiled Rust extension. `pip` downloads a wheel matching your platform, so **no Rust toolchain or compiler is required** on Linux/macOS/Windows with CPython 3.10–3.13. On any platform without a matching wheel, the base package still installs and **automatically falls back to pure Python** — nothing breaks, it's just slower.
|
|
43
|
+
|
|
44
|
+
The accelerator is **transparent** — the same API uses native automatically when it's present:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from odin_prompt_toolkit import NATIVE_AVAILABLE
|
|
48
|
+
print("native" if NATIVE_AVAILABLE else "pure Python")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Force pure Python even when native is installed: `export ODIN_PROMPT_TOOLKIT_NO_NATIVE=1`.
|
|
52
|
+
|
|
53
|
+
### Speed tradeoff
|
|
54
|
+
|
|
55
|
+
Native replaces the hot signature-generation loop with compiled, SIMD-optimized Rust:
|
|
56
|
+
|
|
57
|
+
| | Native (Rust) | Pure Python |
|
|
58
|
+
|---|---|---|
|
|
59
|
+
| Throughput | ~5,300 sigs/sec | ~85 sigs/sec (384-dim) … ~9 sigs/sec (1024-dim) |
|
|
60
|
+
| Per signature | ~0.2 ms | ~12 ms … ~115 ms |
|
|
61
|
+
| vs native | — | **~60×–600× slower** |
|
|
62
|
+
|
|
63
|
+
The multiplier depends on **embedding dimension**: pure Python loops over every dimension, so the gap widens with larger vectors (~63× at 384-dim, ~590× at 1024-dim). Native throughput is roughly constant.
|
|
64
|
+
|
|
65
|
+
**What it means end-to-end:** in a real pipeline, embedding generation usually dominates. On the 3,714-prompt benchmark (local ONNX, CPU) embedding took 112.6 s; adding signature generation cost **0.7 s with native (+0.6%)** vs **43.8 s pure Python (+38%)**. When embeddings are pre-computed or cached (e.g. real-time dedup), signature generation *is* the cost — and the full speedup applies.
|
|
66
|
+
|
|
67
|
+
**Rule of thumb:** use `[native]` in production; pure Python is a correct, always-available fallback for prototyping or unusual platforms.
|
|
68
|
+
|
|
69
|
+
## Quick Start
|
|
70
|
+
|
|
71
|
+
### Basic LSH Signatures
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from odin_prompt_toolkit import simhash_lsh_multi, normalize_vector
|
|
75
|
+
|
|
76
|
+
# Your embedding vector (must be L2-normalized)
|
|
77
|
+
vector = [0.5, 0.5, 0.5, 0.5]
|
|
78
|
+
normalized = normalize_vector(vector)
|
|
79
|
+
|
|
80
|
+
# Generate LSH signatures (3 families, 256 bits, 16 bands)
|
|
81
|
+
families = simhash_lsh_multi(normalized)
|
|
82
|
+
|
|
83
|
+
print(f"Signature: {families[0].signature}")
|
|
84
|
+
print(f"Bands: {families[0].bands}")
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Similarity Comparison
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from odin_prompt_toolkit import simhash_lsh_multi, hamming_distance_hex, cosine_from_hamming
|
|
91
|
+
|
|
92
|
+
# Generate signatures for two vectors
|
|
93
|
+
families1 = simhash_lsh_multi(vector1)
|
|
94
|
+
families2 = simhash_lsh_multi(vector2)
|
|
95
|
+
|
|
96
|
+
# Compute Hamming distance
|
|
97
|
+
distance = hamming_distance_hex(families1[0].signature, families2[0].signature)
|
|
98
|
+
|
|
99
|
+
# Estimate cosine similarity
|
|
100
|
+
similarity = cosine_from_hamming(distance, 256)
|
|
101
|
+
print(f"Estimated cosine similarity: {similarity:.3f}")
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Confidence Matrix LSH (CM-LSH)
|
|
105
|
+
|
|
106
|
+
```python
|
|
107
|
+
from odin_prompt_toolkit.cm_lsh import create_default_cm_lsh
|
|
108
|
+
|
|
109
|
+
# Create CM-LSH hasher (1024 dimensions)
|
|
110
|
+
cm_lsh = create_default_cm_lsh(1024, family=0)
|
|
111
|
+
|
|
112
|
+
# Generate 512-bit signature with confidence matrix
|
|
113
|
+
hash1 = cm_lsh.hash(embedding1)
|
|
114
|
+
hash2 = cm_lsh.hash(embedding2)
|
|
115
|
+
|
|
116
|
+
# Compute calibrated similarity
|
|
117
|
+
similarity = cm_lsh.sim(hash1, hash2)
|
|
118
|
+
print(f"CM-LSH similarity: {similarity:.3f}")
|
|
119
|
+
|
|
120
|
+
# Check for duplicates
|
|
121
|
+
is_duplicate = cm_lsh.is_dup(hash1, hash2, threshold=0.85)
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Signature Versions
|
|
125
|
+
|
|
126
|
+
- **V0**: OpenAI text-embedding-3-large (1536 dimensions, API-based)
|
|
127
|
+
- **V1**: 0din-jailbreak-embeddings-small ONNX (1024 dimensions, local)
|
|
128
|
+
- **Latest**: Resolves to V1
|
|
129
|
+
|
|
130
|
+
**Important**: V0 and V1 signatures are **not comparable** due to different embedding spaces.
|
|
131
|
+
|
|
132
|
+
## Algorithm
|
|
133
|
+
|
|
134
|
+
SimHash via Random Hyperplane LSH (Charikar 2002):
|
|
135
|
+
- Deterministic hyperplanes via SplitMix64 PRNG
|
|
136
|
+
- Default: 3 families × 256 bits × 16 bands
|
|
137
|
+
- Hex-encoded signatures (64 hex chars = 256 bits)
|
|
138
|
+
- Hamming distance → cosine similarity via `cos(π × d/n)`
|
|
139
|
+
|
|
140
|
+
See the [specification](../../spec/SPEC.md) for complete algorithm details.
|
|
141
|
+
|
|
142
|
+
## Development
|
|
143
|
+
|
|
144
|
+
### Setup
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
cd python
|
|
148
|
+
pip install -e ".[dev]"
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### Run Tests
|
|
152
|
+
|
|
153
|
+
```bash
|
|
154
|
+
pytest tests/
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
### Type Checking
|
|
158
|
+
|
|
159
|
+
```bash
|
|
160
|
+
mypy odin_prompt_toolkit/
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
### Formatting
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
black odin_prompt_toolkit/ tests/
|
|
167
|
+
ruff check odin_prompt_toolkit/ tests/
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## License
|
|
171
|
+
|
|
172
|
+
Apache License 2.0
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Basic LSH signature generation example.
|
|
3
|
+
|
|
4
|
+
This example demonstrates:
|
|
5
|
+
- Generating an LSH signature from a normalized vector
|
|
6
|
+
- Default configuration (3 families, 256 bits, 16 bands)
|
|
7
|
+
- Formatting and parsing signature strings
|
|
8
|
+
|
|
9
|
+
Run with: python python/examples/basic_signature.py
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from odin_prompt_toolkit import simhash_lsh_multi
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def main():
|
|
16
|
+
print("=== Basic LSH Signature Generation ===\n")
|
|
17
|
+
|
|
18
|
+
# Example normalized vector (4 dimensions for clarity)
|
|
19
|
+
# In practice, this would come from an embedding model (384 or 1536 dims)
|
|
20
|
+
normalized_vector = [0.5, 0.5, 0.5, 0.5]
|
|
21
|
+
|
|
22
|
+
print(f"Input vector: {normalized_vector}")
|
|
23
|
+
print(f"Vector dimensions: {len(normalized_vector)}\n")
|
|
24
|
+
|
|
25
|
+
# Generate LSH signatures with default configuration
|
|
26
|
+
families_count = 3
|
|
27
|
+
bits = 256
|
|
28
|
+
bands = 16
|
|
29
|
+
|
|
30
|
+
print("Configuration:")
|
|
31
|
+
print(f" Families: {families_count}")
|
|
32
|
+
print(f" Bits per signature: {bits}")
|
|
33
|
+
print(f" Bands: {bands}\n")
|
|
34
|
+
|
|
35
|
+
families = simhash_lsh_multi(normalized_vector, families=families_count, bits=bits, bands=bands)
|
|
36
|
+
|
|
37
|
+
# Display results for each family
|
|
38
|
+
for family in families:
|
|
39
|
+
print(f"Family {family.family}:")
|
|
40
|
+
print(f" Signature (hex): {family.signature}")
|
|
41
|
+
print(f" Signature length: {len(family.signature)} hex chars = {family.bits} bits")
|
|
42
|
+
print(f" Number of bands: {len(family.bands)}")
|
|
43
|
+
print(f" Band 0: {family.bands[0]} (first {len(family.bands[0])} hex chars)")
|
|
44
|
+
print()
|
|
45
|
+
|
|
46
|
+
# Format as 0din signature string (V1 format)
|
|
47
|
+
primary_sig = families[0].signature
|
|
48
|
+
signature_string = f"0din-v1:{primary_sig}"
|
|
49
|
+
|
|
50
|
+
print("Formatted signature string:")
|
|
51
|
+
print(f" {signature_string}")
|
|
52
|
+
print()
|
|
53
|
+
|
|
54
|
+
# In a real application, you would:
|
|
55
|
+
# 1. Store this signature in a database with the original text
|
|
56
|
+
# 2. Use bands for efficient similarity search (LSH indexing)
|
|
57
|
+
# 3. Compare signatures using hamming distance
|
|
58
|
+
|
|
59
|
+
print("✓ Signature generation complete!")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
main()
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Confidence Matrix LSH (CM-LSH) example.
|
|
3
|
+
|
|
4
|
+
This example demonstrates:
|
|
5
|
+
- Enhanced LSH with confidence matrix
|
|
6
|
+
- Dual hash structure (512-bit signature + 512-bit confidence)
|
|
7
|
+
- Backward compatibility with standard LSH (first 256 bits)
|
|
8
|
+
- Calibrated similarity estimation
|
|
9
|
+
|
|
10
|
+
Run with: python python/examples/cm_lsh_example.py
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
from odin_prompt_toolkit.cm_lsh import create_default_cm_lsh
|
|
15
|
+
from odin_prompt_toolkit import normalize_vector
|
|
16
|
+
except ImportError:
|
|
17
|
+
print("This example requires the CM-LSH implementation.")
|
|
18
|
+
print("Install with: pip install -e '.[cm-lsh]'")
|
|
19
|
+
exit(1)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def main():
|
|
23
|
+
print("=== Confidence Matrix LSH (CM-LSH) ===\n")
|
|
24
|
+
|
|
25
|
+
# Example vectors
|
|
26
|
+
vector_a = [1.0, 1.0, 1.0, 1.0]
|
|
27
|
+
vector_b = [1.0, 0.9, 1.1, 1.0] # Similar to A
|
|
28
|
+
vector_c = [-1.0, -1.0, -1.0, -1.0] # Opposite to A
|
|
29
|
+
|
|
30
|
+
print("Input vectors:")
|
|
31
|
+
print(f" A: {vector_a}")
|
|
32
|
+
print(f" B: {vector_b} (similar to A)")
|
|
33
|
+
print(f" C: {vector_c} (opposite to A)\n")
|
|
34
|
+
|
|
35
|
+
# Normalize vectors
|
|
36
|
+
norm_a = normalize_vector(vector_a)
|
|
37
|
+
norm_b = normalize_vector(vector_b)
|
|
38
|
+
norm_c = normalize_vector(vector_c)
|
|
39
|
+
|
|
40
|
+
# Create CM-LSH hasher with default configuration
|
|
41
|
+
# This uses identity ITQ (no learned rotation) for simplicity
|
|
42
|
+
# Family 0 for deterministic results
|
|
43
|
+
hasher = create_default_cm_lsh(len(norm_a), family=0)
|
|
44
|
+
|
|
45
|
+
print("CM-LSH Configuration:")
|
|
46
|
+
print(" Total bits: 512 (256 LSH-TS + 256 ITQ)")
|
|
47
|
+
print(" First 256 bits: LSH-TS compatible")
|
|
48
|
+
print(" Confidence matrix: Alpha-weighted agreement\n")
|
|
49
|
+
|
|
50
|
+
# Generate dual hashes
|
|
51
|
+
hash_a = hasher.hash(norm_a)
|
|
52
|
+
hash_b = hasher.hash(norm_b)
|
|
53
|
+
hash_c = hasher.hash(norm_c)
|
|
54
|
+
|
|
55
|
+
print("Dual hashes (showing first 32 hex chars of 128):")
|
|
56
|
+
print(f" A: hash={hash_a.hash_a[:32]} conf={hash_a.hash_b[:32]}")
|
|
57
|
+
print(f" B: hash={hash_b.hash_a[:32]} conf={hash_b.hash_b[:32]}")
|
|
58
|
+
print(f" C: hash={hash_c.hash_a[:32]} conf={hash_c.hash_b[:32]}")
|
|
59
|
+
print()
|
|
60
|
+
|
|
61
|
+
# Demonstrate LSH-TS backward compatibility
|
|
62
|
+
print("LSH-TS compatibility (first 256 bits):")
|
|
63
|
+
print(f" A: {hash_a.lsh_ts_compat()[:16]}")
|
|
64
|
+
print(f" B: {hash_b.lsh_ts_compat()[:16]}")
|
|
65
|
+
print(f" C: {hash_c.lsh_ts_compat()[:16]}")
|
|
66
|
+
print(" (showing first 16 hex chars of 64)\n")
|
|
67
|
+
|
|
68
|
+
# Compute calibrated similarities
|
|
69
|
+
print("Calibrated similarities:\n")
|
|
70
|
+
|
|
71
|
+
sim_ab = hasher.sim(hash_a, hash_b)
|
|
72
|
+
sim_ac = hasher.sim(hash_a, hash_c)
|
|
73
|
+
sim_bc = hasher.sim(hash_b, hash_c)
|
|
74
|
+
|
|
75
|
+
print(f" A vs B: {sim_ab:.4f}")
|
|
76
|
+
print(f" A vs C: {sim_ac:.4f}")
|
|
77
|
+
print(f" B vs C: {sim_bc:.4f}")
|
|
78
|
+
|
|
79
|
+
print("\n✓ CM-LSH example complete!")
|
|
80
|
+
print("\nKey advantages of CM-LSH:")
|
|
81
|
+
print(" - Confidence matrix weights reliable bits higher")
|
|
82
|
+
print(" - Isotonic calibration improves similarity estimates")
|
|
83
|
+
print(" - Dual hash (LSH + ITQ) for better quantization")
|
|
84
|
+
print(" - Backward compatible with standard LSH")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
if __name__ == "__main__":
|
|
88
|
+
main()
|