blinkthink 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- blinkthink-1.1.0/LICENSE +21 -0
- blinkthink-1.1.0/PKG-INFO +242 -0
- blinkthink-1.1.0/README.md +184 -0
- blinkthink-1.1.0/blinkthink/__init__.py +39 -0
- blinkthink-1.1.0/blinkthink/benchmark.py +103 -0
- blinkthink-1.1.0/blinkthink/calibrator.py +89 -0
- blinkthink-1.1.0/blinkthink/client.py +126 -0
- blinkthink-1.1.0/blinkthink/py.typed +0 -0
- blinkthink-1.1.0/blinkthink/remote.py +181 -0
- blinkthink-1.1.0/blinkthink/router.py +110 -0
- blinkthink-1.1.0/blinkthink/schemas.py +125 -0
- blinkthink-1.1.0/blinkthink/server.py +197 -0
- blinkthink-1.1.0/blinkthink/system1.py +225 -0
- blinkthink-1.1.0/blinkthink/system2.py +408 -0
- blinkthink-1.1.0/blinkthink/tuner.py +185 -0
- blinkthink-1.1.0/blinkthink.egg-info/PKG-INFO +242 -0
- blinkthink-1.1.0/blinkthink.egg-info/SOURCES.txt +22 -0
- blinkthink-1.1.0/blinkthink.egg-info/dependency_links.txt +1 -0
- blinkthink-1.1.0/blinkthink.egg-info/entry_points.txt +2 -0
- blinkthink-1.1.0/blinkthink.egg-info/requires.txt +39 -0
- blinkthink-1.1.0/blinkthink.egg-info/top_level.txt +1 -0
- blinkthink-1.1.0/pyproject.toml +104 -0
- blinkthink-1.1.0/setup.cfg +4 -0
- blinkthink-1.1.0/tests/test_core.py +170 -0
blinkthink-1.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 BlinkThink Authors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: blinkthink
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Calibrated Dual-Process Routing for Open-Weight Language Models
|
|
5
|
+
Author-email: Gabor Petnehazi <gabor.petnehazi@rebuildai.co.uk>
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/RebuildAI-UK/blinkthink
|
|
8
|
+
Project-URL: Documentation, https://github.com/RebuildAI-UK/blinkthink#readme
|
|
9
|
+
Project-URL: Repository, https://github.com/RebuildAI-UK/blinkthink.git
|
|
10
|
+
Keywords: llm,decision-making,dual-process,calibration,system-1-system-2,transformers,routing
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: torch>=2.0.0
|
|
24
|
+
Requires-Dist: transformers>=4.40.0
|
|
25
|
+
Requires-Dist: pydantic>=2.0.0
|
|
26
|
+
Requires-Dist: httpx>=0.24.0
|
|
27
|
+
Requires-Dist: numpy>=1.24.0
|
|
28
|
+
Requires-Dist: scipy>=1.10.0
|
|
29
|
+
Provides-Extra: quantization
|
|
30
|
+
Requires-Dist: bitsandbytes>=0.43.0; extra == "quantization"
|
|
31
|
+
Requires-Dist: accelerate>=0.26.0; extra == "quantization"
|
|
32
|
+
Provides-Extra: server
|
|
33
|
+
Requires-Dist: fastapi>=0.100.0; extra == "server"
|
|
34
|
+
Requires-Dist: uvicorn[standard]>=0.20.0; extra == "server"
|
|
35
|
+
Requires-Dist: accelerate>=0.26.0; extra == "server"
|
|
36
|
+
Requires-Dist: bitsandbytes>=0.43.0; extra == "server"
|
|
37
|
+
Provides-Extra: benchmarks
|
|
38
|
+
Requires-Dist: datasets>=2.14.0; extra == "benchmarks"
|
|
39
|
+
Requires-Dist: pandas>=2.0.0; extra == "benchmarks"
|
|
40
|
+
Requires-Dist: tqdm>=4.65.0; extra == "benchmarks"
|
|
41
|
+
Requires-Dist: accelerate>=0.26.0; extra == "benchmarks"
|
|
42
|
+
Requires-Dist: bitsandbytes>=0.43.0; extra == "benchmarks"
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
45
|
+
Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
|
|
46
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
47
|
+
Requires-Dist: mypy>=1.5.0; extra == "dev"
|
|
48
|
+
Requires-Dist: fastapi>=0.100.0; extra == "dev"
|
|
49
|
+
Provides-Extra: all
|
|
50
|
+
Requires-Dist: fastapi>=0.100.0; extra == "all"
|
|
51
|
+
Requires-Dist: uvicorn[standard]>=0.20.0; extra == "all"
|
|
52
|
+
Requires-Dist: datasets>=2.14.0; extra == "all"
|
|
53
|
+
Requires-Dist: pandas>=2.0.0; extra == "all"
|
|
54
|
+
Requires-Dist: tqdm>=4.65.0; extra == "all"
|
|
55
|
+
Requires-Dist: accelerate>=0.26.0; extra == "all"
|
|
56
|
+
Requires-Dist: bitsandbytes>=0.43.0; extra == "all"
|
|
57
|
+
Dynamic: license-file
|
|
58
|
+
|
|
59
|
+
# BlinkThink
|
|
60
|
+
|
|
61
|
+
[](https://pypi.org/project/blinkthink/)
|
|
62
|
+
[](LICENSE)
|
|
63
|
+
[](https://www.python.org/)
|
|
64
|
+
[](https://pydantic.dev/)
|
|
65
|
+
|
|
66
|
+
**Training-Free Calibrated Dual-Process Routing for Open-Weight Language Models.**
|
|
67
|
+
|
|
68
|
+
BlinkThink implements Daniel Kahneman's **System 1 / System 2** cognitive paradigm for discrete LLM decisions (classification, triage, assertion verification, and multiple-choice QA).
|
|
69
|
+
|
|
70
|
+
- **System 1 (`BlinkEngine`):** Single batched prefill pass with $N$ cyclic option permutations (`0` autoregressively decoded tokens, ~130–240ms on 4B–12B NF4 models).
|
|
71
|
+
- **System 2 (`ThinkEngine`):** Deliberative Chain-of-Thought (CoT) with XML tag delimitation and early exit, triggered selectively only when System 1 confidence falls below a validation-tuned routing threshold $\tau^*$.
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## Architecture Overview
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
┌────────────────────────────┐
|
|
79
|
+
│ Incoming Decision │
|
|
80
|
+
│ (Choice / Binary) │
|
|
81
|
+
└─────────────┬──────────────┘
|
|
82
|
+
│
|
|
83
|
+
▼
|
|
84
|
+
┌────────────────────────────┐
|
|
85
|
+
│ System 1: BlinkEngine │
|
|
86
|
+
│ • N cyclic permutations │
|
|
87
|
+
│ • Single batched prefill │
|
|
88
|
+
│ • Temperature scaling T* │
|
|
89
|
+
└─────────────┬──────────────┘
|
|
90
|
+
│
|
|
91
|
+
Confidence >= tau* ?
|
|
92
|
+
│
|
|
93
|
+
┌──────────────┴──────────────┐
|
|
94
|
+
Yes │ │ No
|
|
95
|
+
(Fast Path) ▼ ▼ (Deliberate Path)
|
|
96
|
+
┌─────────────────────────┐ ┌───────────────────────────┐
|
|
97
|
+
│ Instant Return │ │ System 2: ThinkEngine │
|
|
98
|
+
│ • 0 decoded tokens │ │ • Autoregressive CoT │
|
|
99
|
+
│ • Sub-second prefill │ │ • StopStringCriteria │
|
|
100
|
+
│ • Calibrated probs │ │ • Calibrated token probs │
|
|
101
|
+
└─────────────────────────┘ └─────────────┬─────────────┘
|
|
102
|
+
│
|
|
103
|
+
▼
|
|
104
|
+
┌───────────────────────────┐
|
|
105
|
+
│ Final Decision & Trace │
|
|
106
|
+
└───────────────────────────┘
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Installation
|
|
112
|
+
|
|
113
|
+
Install from PyPI:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
# Core package (local inference & remote HTTP client)
|
|
117
|
+
pip install blinkthink
|
|
118
|
+
|
|
119
|
+
# With 4-bit/8-bit quantization support
|
|
120
|
+
pip install "blinkthink[quantization]"
|
|
121
|
+
|
|
122
|
+
# With FastAPI server, benchmarks, and quantization
|
|
123
|
+
pip install "blinkthink[all]"
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Or install from source for development:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
git clone https://github.com/RebuildAI-UK/blinkthink.git
|
|
130
|
+
cd blinkthink
|
|
131
|
+
pip install -e ".[all,dev]"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## Quick Example
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import torch
|
|
140
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
141
|
+
from blinkthink import BlinkThink
|
|
142
|
+
|
|
143
|
+
model_id = "google/gemma-4-12B-it"
|
|
144
|
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
|
145
|
+
model = AutoModelForCausalLM.from_pretrained(
|
|
146
|
+
model_id,
|
|
147
|
+
torch_dtype=torch.bfloat16,
|
|
148
|
+
device_map="auto"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
client = BlinkThink(model=model, tokenizer=tokenizer, threshold=0.35, n_blinks=2)
|
|
152
|
+
|
|
153
|
+
# 1. Multi-Choice Classification (with optional per-call threshold / blink overrides)
|
|
154
|
+
res = client.choice(
|
|
155
|
+
prompt="Which organelle is responsible for cellular ATP synthesis?",
|
|
156
|
+
options=["Golgi Apparatus", "Mitochondria", "Ribosome", "Endoplasmic Reticulum"],
|
|
157
|
+
n_blinks=2, # Override blink count per-decision
|
|
158
|
+
threshold=0.35, # Override escalation cutoff per-decision
|
|
159
|
+
)
|
|
160
|
+
print(f"Decision: {res.decision}")
|
|
161
|
+
print(f"Confidence: {res.confidence:.3f}")
|
|
162
|
+
print(f"System: {res.system_used} ({res.latency_ms:.1f}ms)")
|
|
163
|
+
|
|
164
|
+
# 2. Binary Assertion Verification
|
|
165
|
+
guard = client.binary(
|
|
166
|
+
prompt="Is light faster in glass than in vacuum?",
|
|
167
|
+
threshold=0.40,
|
|
168
|
+
)
|
|
169
|
+
print(f"Decision: {guard.decision}") # "No"
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
---
|
|
173
|
+
|
|
174
|
+
## Benchmark Highlights
|
|
175
|
+
|
|
176
|
+
Evaluated under 4-bit NF4 quantization on MMLU ($K=4$, $M_{\text{test}}=1000$) and BoolQ ($K=2$, $M_{\text{test}}=800$):
|
|
177
|
+
|
|
178
|
+
| Model | Dataset | Mode | Acc (%) | Latency | Sys 2 Esc (%) |
|
|
179
|
+
| :--- | :--- | :--- | :---: | :---: | :---: |
|
|
180
|
+
| `google/gemma-4-12B-it` | MMLU | CoT Baseline | 79.4 | 57.80 s | 100.0 |
|
|
181
|
+
| `google/gemma-4-12B-it` | MMLU | **BlinkThink ($N=2$, Balanced)** | **78.9** | **19.16 s** (-66.8%) | **28.9** |
|
|
182
|
+
| `google/gemma-4-12B-it` | BoolQ | CoT Baseline | 89.9 | 27.45 s | 100.0 |
|
|
183
|
+
| `google/gemma-4-12B-it` | BoolQ | **BlinkThink ($N=2$, Balanced)** | **89.5** | **3.94 s** (-85.6%) | **8.8** |
|
|
184
|
+
| `Qwen/Qwen3.5-4B` | MMLU | CoT Baseline | 43.7 | 46.15 s | 100.0 |
|
|
185
|
+
| `Qwen/Qwen3.5-4B` | MMLU | **BlinkThink ($N=4$, Cyclic)** | **68.4** | **190.4 ms** | **0.0** |
|
|
186
|
+
| `Qwen/Qwen3.5-4B` | BoolQ | CoT Baseline | 86.4 | 44.48 s | 100.0 |
|
|
187
|
+
| `Qwen/Qwen3.5-4B` | BoolQ | **BlinkThink ($N=2$, Pure S1)** | **86.1** | **149.1 ms** | **0.0** |
|
|
188
|
+
|
|
189
|
+
---
|
|
190
|
+
|
|
191
|
+
## Guides & Documentation
|
|
192
|
+
|
|
193
|
+
- **[Quickstart Guide](docs/QUICKSTART.md):** Setup, local usage, and production REST API deployment.
|
|
194
|
+
- **[Decision API Guide](docs/DECISION_API_GUIDE.md):** Mathematical formulas, permutation invariance, temperature calibration, and optimal $\tau^*$ routing theory.
|
|
195
|
+
- **[Examples Directory](examples/):**
|
|
196
|
+
- `01_in_process_sdk.py`: In-memory GPU SDK usage.
|
|
197
|
+
- `02_fastapi_server.py`: FastAPI OpenAPI REST deployment.
|
|
198
|
+
- `03_remote_client.py`: Sync & Async HTTP client usage.
|
|
199
|
+
|
|
200
|
+
---
|
|
201
|
+
|
|
202
|
+
## Running Benchmarks
|
|
203
|
+
|
|
204
|
+
Run the built-in study harness across curated or standard Hugging Face datasets:
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
# 1. Single study run (fast curated sanity test)
|
|
208
|
+
python benchmarks/run_study.py --dataset curated
|
|
209
|
+
|
|
210
|
+
# 2. Single study run on MMLU
|
|
211
|
+
python benchmarks/run_study.py --dataset mmlu --val_size 50 --test_size 200 --tune_objective balanced
|
|
212
|
+
|
|
213
|
+
# 3. Run complete matrix across default models and all objectives
|
|
214
|
+
python benchmarks/run_study_matrix.py --run --dataset curated
|
|
215
|
+
|
|
216
|
+
# 4. Run matrix across custom models on MMLU
|
|
217
|
+
python benchmarks/run_study_matrix.py --run --models Qwen/Qwen3.5-4B google/gemma-4-12B-it --dataset mmlu
|
|
218
|
+
|
|
219
|
+
# 5. Inspect and compare existing results (auto-detects datasets)
|
|
220
|
+
python benchmarks/run_study_matrix.py
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Citation
|
|
226
|
+
|
|
227
|
+
If you use BlinkThink in your research, please cite:
|
|
228
|
+
|
|
229
|
+
```bibtex
|
|
230
|
+
@article{petnehazi2026blinkthink,
|
|
231
|
+
title={BlinkThink: Training-Free Calibrated Dual-Process Routing for Discrete LLM Decisions},
|
|
232
|
+
author={Petnehazi, Gabor},
|
|
233
|
+
year={2026},
|
|
234
|
+
url={https://github.com/RebuildAI-UK/blinkthink}
|
|
235
|
+
}
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
---
|
|
239
|
+
|
|
240
|
+
## License
|
|
241
|
+
|
|
242
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# BlinkThink
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/blinkthink/)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
[](https://www.python.org/)
|
|
6
|
+
[](https://pydantic.dev/)
|
|
7
|
+
|
|
8
|
+
**Training-Free Calibrated Dual-Process Routing for Open-Weight Language Models.**
|
|
9
|
+
|
|
10
|
+
BlinkThink implements Daniel Kahneman's **System 1 / System 2** cognitive paradigm for discrete LLM decisions (classification, triage, assertion verification, and multiple-choice QA).
|
|
11
|
+
|
|
12
|
+
- **System 1 (`BlinkEngine`):** Single batched prefill pass with $N$ cyclic option permutations (`0` autoregressively decoded tokens, ~130–240ms on 4B–12B NF4 models).
|
|
13
|
+
- **System 2 (`ThinkEngine`):** Deliberative Chain-of-Thought (CoT) with XML tag delimitation and early exit, triggered selectively only when System 1 confidence falls below a validation-tuned routing threshold $\tau^*$.
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## Architecture Overview
|
|
18
|
+
|
|
19
|
+
```
|
|
20
|
+
┌────────────────────────────┐
|
|
21
|
+
│ Incoming Decision │
|
|
22
|
+
│ (Choice / Binary) │
|
|
23
|
+
└─────────────┬──────────────┘
|
|
24
|
+
│
|
|
25
|
+
▼
|
|
26
|
+
┌────────────────────────────┐
|
|
27
|
+
│ System 1: BlinkEngine │
|
|
28
|
+
│ • N cyclic permutations │
|
|
29
|
+
│ • Single batched prefill │
|
|
30
|
+
│ • Temperature scaling T* │
|
|
31
|
+
└─────────────┬──────────────┘
|
|
32
|
+
│
|
|
33
|
+
Confidence >= tau* ?
|
|
34
|
+
│
|
|
35
|
+
┌──────────────┴──────────────┐
|
|
36
|
+
Yes │ │ No
|
|
37
|
+
(Fast Path) ▼ ▼ (Deliberate Path)
|
|
38
|
+
┌─────────────────────────┐ ┌───────────────────────────┐
|
|
39
|
+
│ Instant Return │ │ System 2: ThinkEngine │
|
|
40
|
+
│ • 0 decoded tokens │ │ • Autoregressive CoT │
|
|
41
|
+
│ • Sub-second prefill │ │ • StopStringCriteria │
|
|
42
|
+
│ • Calibrated probs │ │ • Calibrated token probs │
|
|
43
|
+
└─────────────────────────┘ └─────────────┬─────────────┘
|
|
44
|
+
│
|
|
45
|
+
▼
|
|
46
|
+
┌───────────────────────────┐
|
|
47
|
+
│ Final Decision & Trace │
|
|
48
|
+
└───────────────────────────┘
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Installation
|
|
54
|
+
|
|
55
|
+
Install from PyPI:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
# Core package (local inference & remote HTTP client)
|
|
59
|
+
pip install blinkthink
|
|
60
|
+
|
|
61
|
+
# With 4-bit/8-bit quantization support
|
|
62
|
+
pip install "blinkthink[quantization]"
|
|
63
|
+
|
|
64
|
+
# With FastAPI server, benchmarks, and quantization
|
|
65
|
+
pip install "blinkthink[all]"
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
Or install from source for development:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
git clone https://github.com/RebuildAI-UK/blinkthink.git
|
|
72
|
+
cd blinkthink
|
|
73
|
+
pip install -e ".[all,dev]"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Quick Example
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import torch
|
|
82
|
+
from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
83
|
+
from blinkthink import BlinkThink
|
|
84
|
+
|
|
85
|
+
model_id = "google/gemma-4-12B-it"
|
|
86
|
+
tokenizer = AutoTokenizer.from_pretrained(model_id)
|
|
87
|
+
model = AutoModelForCausalLM.from_pretrained(
|
|
88
|
+
model_id,
|
|
89
|
+
torch_dtype=torch.bfloat16,
|
|
90
|
+
device_map="auto"
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
client = BlinkThink(model=model, tokenizer=tokenizer, threshold=0.35, n_blinks=2)
|
|
94
|
+
|
|
95
|
+
# 1. Multi-Choice Classification (with optional per-call threshold / blink overrides)
|
|
96
|
+
res = client.choice(
|
|
97
|
+
prompt="Which organelle is responsible for cellular ATP synthesis?",
|
|
98
|
+
options=["Golgi Apparatus", "Mitochondria", "Ribosome", "Endoplasmic Reticulum"],
|
|
99
|
+
n_blinks=2, # Override blink count per-decision
|
|
100
|
+
threshold=0.35, # Override escalation cutoff per-decision
|
|
101
|
+
)
|
|
102
|
+
print(f"Decision: {res.decision}")
|
|
103
|
+
print(f"Confidence: {res.confidence:.3f}")
|
|
104
|
+
print(f"System: {res.system_used} ({res.latency_ms:.1f}ms)")
|
|
105
|
+
|
|
106
|
+
# 2. Binary Assertion Verification
|
|
107
|
+
guard = client.binary(
|
|
108
|
+
prompt="Is light faster in glass than in vacuum?",
|
|
109
|
+
threshold=0.40,
|
|
110
|
+
)
|
|
111
|
+
print(f"Decision: {guard.decision}") # "No"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Benchmark Highlights
|
|
117
|
+
|
|
118
|
+
Evaluated under 4-bit NF4 quantization on MMLU ($K=4$, $M_{\text{test}}=1000$) and BoolQ ($K=2$, $M_{\text{test}}=800$):
|
|
119
|
+
|
|
120
|
+
| Model | Dataset | Mode | Acc (%) | Latency | Sys 2 Esc (%) |
|
|
121
|
+
| :--- | :--- | :--- | :---: | :---: | :---: |
|
|
122
|
+
| `google/gemma-4-12B-it` | MMLU | CoT Baseline | 79.4 | 57.80 s | 100.0 |
|
|
123
|
+
| `google/gemma-4-12B-it` | MMLU | **BlinkThink ($N=2$, Balanced)** | **78.9** | **19.16 s** (-66.8%) | **28.9** |
|
|
124
|
+
| `google/gemma-4-12B-it` | BoolQ | CoT Baseline | 89.9 | 27.45 s | 100.0 |
|
|
125
|
+
| `google/gemma-4-12B-it` | BoolQ | **BlinkThink ($N=2$, Balanced)** | **89.5** | **3.94 s** (-85.6%) | **8.8** |
|
|
126
|
+
| `Qwen/Qwen3.5-4B` | MMLU | CoT Baseline | 43.7 | 46.15 s | 100.0 |
|
|
127
|
+
| `Qwen/Qwen3.5-4B` | MMLU | **BlinkThink ($N=4$, Cyclic)** | **68.4** | **190.4 ms** | **0.0** |
|
|
128
|
+
| `Qwen/Qwen3.5-4B` | BoolQ | CoT Baseline | 86.4 | 44.48 s | 100.0 |
|
|
129
|
+
| `Qwen/Qwen3.5-4B` | BoolQ | **BlinkThink ($N=2$, Pure S1)** | **86.1** | **149.1 ms** | **0.0** |
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## Guides & Documentation
|
|
134
|
+
|
|
135
|
+
- **[Quickstart Guide](docs/QUICKSTART.md):** Setup, local usage, and production REST API deployment.
|
|
136
|
+
- **[Decision API Guide](docs/DECISION_API_GUIDE.md):** Mathematical formulas, permutation invariance, temperature calibration, and optimal $\tau^*$ routing theory.
|
|
137
|
+
- **[Examples Directory](examples/):**
|
|
138
|
+
- `01_in_process_sdk.py`: In-memory GPU SDK usage.
|
|
139
|
+
- `02_fastapi_server.py`: FastAPI OpenAPI REST deployment.
|
|
140
|
+
- `03_remote_client.py`: Sync & Async HTTP client usage.
|
|
141
|
+
|
|
142
|
+
---
|
|
143
|
+
|
|
144
|
+
## Running Benchmarks
|
|
145
|
+
|
|
146
|
+
Run the built-in study harness across curated or standard Hugging Face datasets:
|
|
147
|
+
|
|
148
|
+
```bash
|
|
149
|
+
# 1. Single study run (fast curated sanity test)
|
|
150
|
+
python benchmarks/run_study.py --dataset curated
|
|
151
|
+
|
|
152
|
+
# 2. Single study run on MMLU
|
|
153
|
+
python benchmarks/run_study.py --dataset mmlu --val_size 50 --test_size 200 --tune_objective balanced
|
|
154
|
+
|
|
155
|
+
# 3. Run complete matrix across default models and all objectives
|
|
156
|
+
python benchmarks/run_study_matrix.py --run --dataset curated
|
|
157
|
+
|
|
158
|
+
# 4. Run matrix across custom models on MMLU
|
|
159
|
+
python benchmarks/run_study_matrix.py --run --models Qwen/Qwen3.5-4B google/gemma-4-12B-it --dataset mmlu
|
|
160
|
+
|
|
161
|
+
# 5. Inspect and compare existing results (auto-detects datasets)
|
|
162
|
+
python benchmarks/run_study_matrix.py
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## Citation
|
|
168
|
+
|
|
169
|
+
If you use BlinkThink in your research, please cite:
|
|
170
|
+
|
|
171
|
+
```bibtex
|
|
172
|
+
@article{petnehazi2026blinkthink,
|
|
173
|
+
title={BlinkThink: Training-Free Calibrated Dual-Process Routing for Discrete LLM Decisions},
|
|
174
|
+
author={Petnehazi, Gabor},
|
|
175
|
+
year={2026},
|
|
176
|
+
url={https://github.com/RebuildAI-UK/blinkthink}
|
|
177
|
+
}
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
---
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
BlinkThink: Calibrated Dual-Process Routing for Open-Weight Language Models.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .benchmark import BenchmarkReport, BlinkStudyEvaluator, compute_ece
|
|
6
|
+
from .calibrator import TemperatureCalibrator
|
|
7
|
+
from .client import BlinkThink
|
|
8
|
+
from .remote import AsyncBlinkThinkRemoteClient, BlinkThinkRemoteClient
|
|
9
|
+
from .router import BlinkThinkRouter
|
|
10
|
+
from .system1 import BlinkEngine
|
|
11
|
+
from .system2 import ThinkEngine
|
|
12
|
+
from .tuner import BlinkThinkTuner, TuningReport
|
|
13
|
+
from .schemas import (
|
|
14
|
+
BlinkTelemetry,
|
|
15
|
+
DecisionRequest,
|
|
16
|
+
DecisionResult,
|
|
17
|
+
DecisionType,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__version__ = "1.1.0"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"BlinkThink",
|
|
24
|
+
"BlinkThinkRemoteClient",
|
|
25
|
+
"AsyncBlinkThinkRemoteClient",
|
|
26
|
+
"BlinkThinkRouter",
|
|
27
|
+
"BlinkEngine",
|
|
28
|
+
"ThinkEngine",
|
|
29
|
+
"BlinkThinkTuner",
|
|
30
|
+
"TemperatureCalibrator",
|
|
31
|
+
"BlinkStudyEvaluator",
|
|
32
|
+
"DecisionRequest",
|
|
33
|
+
"DecisionResult",
|
|
34
|
+
"DecisionType",
|
|
35
|
+
"BlinkTelemetry",
|
|
36
|
+
"BenchmarkReport",
|
|
37
|
+
"TuningReport",
|
|
38
|
+
"compute_ece",
|
|
39
|
+
]
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from dataclasses import asdict, dataclass
|
|
2
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
3
|
+
import numpy as np
|
|
4
|
+
from .router import BlinkThinkRouter
|
|
5
|
+
from .schemas import DecisionRequest
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class BenchmarkReport:
|
|
10
|
+
config_name: str
|
|
11
|
+
n_blinks: int
|
|
12
|
+
threshold: float
|
|
13
|
+
accuracy: float
|
|
14
|
+
avg_latency_ms: float
|
|
15
|
+
system2_escalation_pct: float
|
|
16
|
+
expected_calibration_error: float
|
|
17
|
+
avg_drift: float
|
|
18
|
+
avg_consensus: float
|
|
19
|
+
|
|
20
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
21
|
+
return asdict(self)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def compute_ece(probs: np.ndarray, labels: np.ndarray, n_bins: int = 10) -> float:
|
|
25
|
+
"""
|
|
26
|
+
Computes Expected Calibration Error (ECE).
|
|
27
|
+
"""
|
|
28
|
+
if len(probs) == 0:
|
|
29
|
+
return 0.0
|
|
30
|
+
probs = np.clip(probs, 0.0, 1.0)
|
|
31
|
+
bin_boundaries = np.linspace(0, 1, n_bins + 1)
|
|
32
|
+
ece = 0.0
|
|
33
|
+
for i in range(n_bins):
|
|
34
|
+
bin_lower, bin_upper = bin_boundaries[i], bin_boundaries[i + 1]
|
|
35
|
+
in_bin = (
|
|
36
|
+
(probs >= bin_lower) & (probs <= bin_upper)
|
|
37
|
+
if i == 0
|
|
38
|
+
else (probs > bin_lower) & (probs <= bin_upper)
|
|
39
|
+
)
|
|
40
|
+
prop_in_bin = np.mean(in_bin)
|
|
41
|
+
if prop_in_bin > 0:
|
|
42
|
+
accuracy_in_bin = np.mean(labels[in_bin])
|
|
43
|
+
avg_confidence_in_bin = np.mean(probs[in_bin])
|
|
44
|
+
ece += np.abs(accuracy_in_bin - avg_confidence_in_bin) * prop_in_bin
|
|
45
|
+
return float(ece)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class BlinkStudyEvaluator:
|
|
49
|
+
def __init__(self, router: BlinkThinkRouter):
|
|
50
|
+
self.router = router
|
|
51
|
+
|
|
52
|
+
def evaluate_dataset(
|
|
53
|
+
self,
|
|
54
|
+
dataset: List[Tuple[DecisionRequest, Any]],
|
|
55
|
+
config_name: str = "BlinkThink",
|
|
56
|
+
n_blinks: int = 2,
|
|
57
|
+
threshold: float = 0.35,
|
|
58
|
+
force_mode: Optional[str] = None,
|
|
59
|
+
) -> BenchmarkReport:
|
|
60
|
+
correct: List[int] = []
|
|
61
|
+
latencies: List[float] = []
|
|
62
|
+
escalated: List[int] = []
|
|
63
|
+
confidences: List[float] = []
|
|
64
|
+
drifts: List[float] = []
|
|
65
|
+
consensuses: List[float] = []
|
|
66
|
+
|
|
67
|
+
for req, y_true in dataset:
|
|
68
|
+
res = self.router.decide(
|
|
69
|
+
req,
|
|
70
|
+
n_blinks=n_blinks,
|
|
71
|
+
threshold=threshold,
|
|
72
|
+
force_mode=force_mode,
|
|
73
|
+
)
|
|
74
|
+
is_correct = (res.decision == y_true)
|
|
75
|
+
|
|
76
|
+
correct.append(1 if is_correct else 0)
|
|
77
|
+
latencies.append(res.latency_ms)
|
|
78
|
+
escalated.append(1 if res.system_used in ("system_2", "system_1_fallback") else 0)
|
|
79
|
+
|
|
80
|
+
top_prob = (
|
|
81
|
+
res.probabilities.get(res.decision, res.confidence)
|
|
82
|
+
if res.probabilities
|
|
83
|
+
else res.confidence
|
|
84
|
+
)
|
|
85
|
+
confidences.append(top_prob)
|
|
86
|
+
|
|
87
|
+
if res.telemetry:
|
|
88
|
+
drifts.append(res.telemetry.drift)
|
|
89
|
+
consensuses.append(res.telemetry.consensus_rate)
|
|
90
|
+
|
|
91
|
+
ece = compute_ece(np.array(confidences), np.array(correct))
|
|
92
|
+
|
|
93
|
+
return BenchmarkReport(
|
|
94
|
+
config_name=config_name,
|
|
95
|
+
n_blinks=n_blinks,
|
|
96
|
+
threshold=threshold,
|
|
97
|
+
accuracy=float(np.mean(correct)) if correct else 0.0,
|
|
98
|
+
avg_latency_ms=float(np.mean(latencies)) if latencies else 0.0,
|
|
99
|
+
system2_escalation_pct=float(np.mean(escalated) * 100.0) if escalated else 0.0,
|
|
100
|
+
expected_calibration_error=ece,
|
|
101
|
+
avg_drift=float(np.mean(drifts)) if drifts else 0.0,
|
|
102
|
+
avg_consensus=float(np.mean(consensuses)) if consensuses else 1.0,
|
|
103
|
+
)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any, Dict, Optional, Tuple
|
|
3
|
+
from scipy.optimize import minimize_scalar
|
|
4
|
+
import torch
|
|
5
|
+
import torch.nn.functional as F
|
|
6
|
+
from .schemas import DecisionType
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TemperatureCalibrator:
|
|
12
|
+
"""
|
|
13
|
+
Manages and optimizes temperature scaling for choice decisions.
|
|
14
|
+
"""
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
temperature: float = 3.4,
|
|
18
|
+
temperature_choice: Optional[float] = None,
|
|
19
|
+
enable_prior_correction: bool = False,
|
|
20
|
+
**kwargs: Any,
|
|
21
|
+
):
|
|
22
|
+
temp = temperature_choice if temperature_choice is not None else temperature
|
|
23
|
+
self.temperature = float(temp)
|
|
24
|
+
self.enable_prior_correction = enable_prior_correction
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def temperatures(self) -> Dict[DecisionType, float]:
|
|
28
|
+
return {DecisionType.CHOICE: self.temperature}
|
|
29
|
+
|
|
30
|
+
def calibrate(self, logits: torch.Tensor, decision_type: Optional[DecisionType] = None) -> torch.Tensor:
|
|
31
|
+
temp = max(self.temperature, 1e-4)
|
|
32
|
+
scaled = logits / temp
|
|
33
|
+
return F.softmax(scaled, dim=-1)
|
|
34
|
+
|
|
35
|
+
def fit(
|
|
36
|
+
self,
|
|
37
|
+
candidate_logits: Any,
|
|
38
|
+
target_indices: Any,
|
|
39
|
+
decision_type: Optional[DecisionType] = None,
|
|
40
|
+
temp_bounds: Tuple[float, float] = (0.1, 10.0),
|
|
41
|
+
) -> float:
|
|
42
|
+
"""
|
|
43
|
+
Finds optimal temperature T* by minimizing Negative Log-Likelihood (NLL).
|
|
44
|
+
"""
|
|
45
|
+
if isinstance(candidate_logits, torch.Tensor) and candidate_logits.ndim == 2:
|
|
46
|
+
logits_list = [row for row in candidate_logits.detach().cpu().float()]
|
|
47
|
+
else:
|
|
48
|
+
logits_list = [
|
|
49
|
+
row.detach().cpu().float() if isinstance(row, torch.Tensor)
|
|
50
|
+
else torch.tensor(row, dtype=torch.float32)
|
|
51
|
+
for row in candidate_logits
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
targets_list = (
|
|
55
|
+
target_indices.detach().cpu().long().tolist()
|
|
56
|
+
if isinstance(target_indices, torch.Tensor)
|
|
57
|
+
else [int(t) for t in target_indices]
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if len(logits_list) < 3:
|
|
61
|
+
logger.warning(
|
|
62
|
+
"Only %d samples provided for calibration. Retaining temperature T=%.2f.",
|
|
63
|
+
len(logits_list),
|
|
64
|
+
self.temperature,
|
|
65
|
+
)
|
|
66
|
+
return self.temperature
|
|
67
|
+
|
|
68
|
+
def nll_objective(t: float) -> float:
|
|
69
|
+
t_clamped = max(t, 1e-4)
|
|
70
|
+
total_nll = 0.0
|
|
71
|
+
for l_row, target in zip(logits_list, targets_list):
|
|
72
|
+
scaled = l_row / t_clamped
|
|
73
|
+
log_probs = F.log_softmax(scaled, dim=-1)
|
|
74
|
+
total_nll += -log_probs[target].item()
|
|
75
|
+
return total_nll / len(logits_list)
|
|
76
|
+
|
|
77
|
+
res = minimize_scalar(nll_objective, bounds=temp_bounds, method="bounded")
|
|
78
|
+
self.temperature = float(res.x)
|
|
79
|
+
logger.info("Temperature calibrated: T* = %.3f", self.temperature)
|
|
80
|
+
return self.temperature
|
|
81
|
+
|
|
82
|
+
def to_dict(self) -> Dict[str, float]:
|
|
83
|
+
return {"choice": self.temperature}
|
|
84
|
+
|
|
85
|
+
def load_dict(self, data: Dict[str, float]):
|
|
86
|
+
if "choice" in data:
|
|
87
|
+
self.temperature = float(data["choice"])
|
|
88
|
+
elif "temperature" in data:
|
|
89
|
+
self.temperature = float(data["temperature"])
|