blinkthink 1.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 BlinkThink Authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,242 @@
1
+ Metadata-Version: 2.4
2
+ Name: blinkthink
3
+ Version: 1.1.0
4
+ Summary: Calibrated Dual-Process Routing for Open-Weight Language Models
5
+ Author-email: Gabor Petnehazi <gabor.petnehazi@rebuildai.co.uk>
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/RebuildAI-UK/blinkthink
8
+ Project-URL: Documentation, https://github.com/RebuildAI-UK/blinkthink#readme
9
+ Project-URL: Repository, https://github.com/RebuildAI-UK/blinkthink.git
10
+ Keywords: llm,decision-making,dual-process,calibration,system-1-system-2,transformers,routing
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
20
+ Requires-Python: >=3.10
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: torch>=2.0.0
24
+ Requires-Dist: transformers>=4.40.0
25
+ Requires-Dist: pydantic>=2.0.0
26
+ Requires-Dist: httpx>=0.24.0
27
+ Requires-Dist: numpy>=1.24.0
28
+ Requires-Dist: scipy>=1.10.0
29
+ Provides-Extra: quantization
30
+ Requires-Dist: bitsandbytes>=0.43.0; extra == "quantization"
31
+ Requires-Dist: accelerate>=0.26.0; extra == "quantization"
32
+ Provides-Extra: server
33
+ Requires-Dist: fastapi>=0.100.0; extra == "server"
34
+ Requires-Dist: uvicorn[standard]>=0.20.0; extra == "server"
35
+ Requires-Dist: accelerate>=0.26.0; extra == "server"
36
+ Requires-Dist: bitsandbytes>=0.43.0; extra == "server"
37
+ Provides-Extra: benchmarks
38
+ Requires-Dist: datasets>=2.14.0; extra == "benchmarks"
39
+ Requires-Dist: pandas>=2.0.0; extra == "benchmarks"
40
+ Requires-Dist: tqdm>=4.65.0; extra == "benchmarks"
41
+ Requires-Dist: accelerate>=0.26.0; extra == "benchmarks"
42
+ Requires-Dist: bitsandbytes>=0.43.0; extra == "benchmarks"
43
+ Provides-Extra: dev
44
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
45
+ Requires-Dist: pytest-asyncio>=0.21.0; extra == "dev"
46
+ Requires-Dist: ruff>=0.1.0; extra == "dev"
47
+ Requires-Dist: mypy>=1.5.0; extra == "dev"
48
+ Requires-Dist: fastapi>=0.100.0; extra == "dev"
49
+ Provides-Extra: all
50
+ Requires-Dist: fastapi>=0.100.0; extra == "all"
51
+ Requires-Dist: uvicorn[standard]>=0.20.0; extra == "all"
52
+ Requires-Dist: datasets>=2.14.0; extra == "all"
53
+ Requires-Dist: pandas>=2.0.0; extra == "all"
54
+ Requires-Dist: tqdm>=4.65.0; extra == "all"
55
+ Requires-Dist: accelerate>=0.26.0; extra == "all"
56
+ Requires-Dist: bitsandbytes>=0.43.0; extra == "all"
57
+ Dynamic: license-file
58
+
59
+ # BlinkThink
60
+
61
+ [![PyPI version](https://img.shields.io/pypi/v/blinkthink.svg)](https://pypi.org/project/blinkthink/)
62
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
63
+ [![Python](https://img.shields.io/badge/Python-3.10%2B-blue)](https://www.python.org/)
64
+ [![Pydantic](https://img.shields.io/badge/Pydantic-v2-green)](https://pydantic.dev/)
65
+
66
+ **Training-Free Calibrated Dual-Process Routing for Open-Weight Language Models.**
67
+
68
+ BlinkThink implements Daniel Kahneman's **System 1 / System 2** cognitive paradigm for discrete LLM decisions (classification, triage, assertion verification, and multiple-choice QA).
69
+
70
+ - **System 1 (`BlinkEngine`):** Single batched prefill pass with $N$ cyclic option permutations (`0` autoregressively decoded tokens, ~130–240ms on 4B–12B NF4 models).
71
+ - **System 2 (`ThinkEngine`):** Deliberative Chain-of-Thought (CoT) with XML tag delimitation and early exit, triggered selectively only when System 1 confidence falls below a validation-tuned routing threshold $\tau^*$.
72
+
73
+ ---
74
+
75
+ ## Architecture Overview
76
+
77
+ ```
78
+ ┌────────────────────────────┐
79
+ │ Incoming Decision │
80
+ │ (Choice / Binary) │
81
+ └─────────────┬──────────────┘
82
+ │
83
+ ▼
84
+ ┌────────────────────────────┐
85
+ │ System 1: BlinkEngine │
86
+ │ • N cyclic permutations │
87
+ │ • Single batched prefill │
88
+ │ • Temperature scaling T* │
89
+ └─────────────┬──────────────┘
90
+ │
91
+ Confidence >= tau* ?
92
+ │
93
+ ┌──────────────┴──────────────┐
94
+ Yes │ │ No
95
+ (Fast Path) ▼ ▼ (Deliberate Path)
96
+ ┌─────────────────────────┐ ┌───────────────────────────┐
97
+ │ Instant Return │ │ System 2: ThinkEngine │
98
+ │ • 0 decoded tokens │ │ • Autoregressive CoT │
99
+ │ • Sub-second prefill │ │ • StopStringCriteria │
100
+ │ • Calibrated probs │ │ • Calibrated token probs │
101
+ └─────────────────────────┘ └─────────────┬─────────────┘
102
+ │
103
+ ▼
104
+ ┌───────────────────────────┐
105
+ │ Final Decision & Trace │
106
+ └───────────────────────────┘
107
+ ```
108
+
109
+ ---
110
+
111
+ ## Installation
112
+
113
+ Install from PyPI:
114
+
115
+ ```bash
116
+ # Core package (local inference & remote HTTP client)
117
+ pip install blinkthink
118
+
119
+ # With 4-bit/8-bit quantization support
120
+ pip install "blinkthink[quantization]"
121
+
122
+ # With FastAPI server, benchmarks, and quantization
123
+ pip install "blinkthink[all]"
124
+ ```
125
+
126
+ Or install from source for development:
127
+
128
+ ```bash
129
+ git clone https://github.com/RebuildAI-UK/blinkthink.git
130
+ cd blinkthink
131
+ pip install -e ".[all,dev]"
132
+ ```
133
+
134
+ ---
135
+
136
+ ## Quick Example
137
+
138
+ ```python
139
+ import torch
140
+ from transformers import AutoModelForCausalLM, AutoTokenizer
141
+ from blinkthink import BlinkThink
142
+
143
+ model_id = "google/gemma-4-12B-it"
144
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
145
+ model = AutoModelForCausalLM.from_pretrained(
146
+ model_id,
147
+ torch_dtype=torch.bfloat16,
148
+ device_map="auto"
149
+ )
150
+
151
+ client = BlinkThink(model=model, tokenizer=tokenizer, threshold=0.35, n_blinks=2)
152
+
153
+ # 1. Multi-Choice Classification (with optional per-call threshold / blink overrides)
154
+ res = client.choice(
155
+ prompt="Which organelle is responsible for cellular ATP synthesis?",
156
+ options=["Golgi Apparatus", "Mitochondria", "Ribosome", "Endoplasmic Reticulum"],
157
+ n_blinks=2, # Override blink count per-decision
158
+ threshold=0.35, # Override escalation cutoff per-decision
159
+ )
160
+ print(f"Decision: {res.decision}")
161
+ print(f"Confidence: {res.confidence:.3f}")
162
+ print(f"System: {res.system_used} ({res.latency_ms:.1f}ms)")
163
+
164
+ # 2. Binary Assertion Verification
165
+ guard = client.binary(
166
+ prompt="Is light faster in glass than in vacuum?",
167
+ threshold=0.40,
168
+ )
169
+ print(f"Decision: {guard.decision}") # "No"
170
+ ```
171
+
172
+ ---
173
+
174
+ ## Benchmark Highlights
175
+
176
+ Evaluated under 4-bit NF4 quantization on MMLU ($K=4$, $M_{\text{test}}=1000$) and BoolQ ($K=2$, $M_{\text{test}}=800$):
177
+
178
+ | Model | Dataset | Mode | Acc (%) | Latency | Sys 2 Esc (%) |
179
+ | :--- | :--- | :--- | :---: | :---: | :---: |
180
+ | `google/gemma-4-12B-it` | MMLU | CoT Baseline | 79.4 | 57.80 s | 100.0 |
181
+ | `google/gemma-4-12B-it` | MMLU | **BlinkThink ($N=2$, Balanced)** | **78.9** | **19.16 s** (-66.8%) | **28.9** |
182
+ | `google/gemma-4-12B-it` | BoolQ | CoT Baseline | 89.9 | 27.45 s | 100.0 |
183
+ | `google/gemma-4-12B-it` | BoolQ | **BlinkThink ($N=2$, Balanced)** | **89.5** | **3.94 s** (-85.6%) | **8.8** |
184
+ | `Qwen/Qwen3.5-4B` | MMLU | CoT Baseline | 43.7 | 46.15 s | 100.0 |
185
+ | `Qwen/Qwen3.5-4B` | MMLU | **BlinkThink ($N=4$, Cyclic)** | **68.4** | **190.4 ms** | **0.0** |
186
+ | `Qwen/Qwen3.5-4B` | BoolQ | CoT Baseline | 86.4 | 44.48 s | 100.0 |
187
+ | `Qwen/Qwen3.5-4B` | BoolQ | **BlinkThink ($N=2$, Pure S1)** | **86.1** | **149.1 ms** | **0.0** |
188
+
189
+ ---
190
+
191
+ ## Guides & Documentation
192
+
193
+ - **[Quickstart Guide](docs/QUICKSTART.md):** Setup, local usage, and production REST API deployment.
194
+ - **[Decision API Guide](docs/DECISION_API_GUIDE.md):** Mathematical formulas, permutation invariance, temperature calibration, and optimal $\tau^*$ routing theory.
195
+ - **[Examples Directory](examples/):**
196
+ - `01_in_process_sdk.py`: In-memory GPU SDK usage.
197
+ - `02_fastapi_server.py`: FastAPI OpenAPI REST deployment.
198
+ - `03_remote_client.py`: Sync & Async HTTP client usage.
199
+
200
+ ---
201
+
202
+ ## Running Benchmarks
203
+
204
+ Run the built-in study harness across curated or standard Hugging Face datasets:
205
+
206
+ ```bash
207
+ # 1. Single study run (fast curated sanity test)
208
+ python benchmarks/run_study.py --dataset curated
209
+
210
+ # 2. Single study run on MMLU
211
+ python benchmarks/run_study.py --dataset mmlu --val_size 50 --test_size 200 --tune_objective balanced
212
+
213
+ # 3. Run complete matrix across default models and all objectives
214
+ python benchmarks/run_study_matrix.py --run --dataset curated
215
+
216
+ # 4. Run matrix across custom models on MMLU
217
+ python benchmarks/run_study_matrix.py --run --models Qwen/Qwen3.5-4B google/gemma-4-12B-it --dataset mmlu
218
+
219
+ # 5. Inspect and compare existing results (auto-detects datasets)
220
+ python benchmarks/run_study_matrix.py
221
+ ```
222
+
223
+ ---
224
+
225
+ ## Citation
226
+
227
+ If you use BlinkThink in your research, please cite:
228
+
229
+ ```bibtex
230
+ @article{petnehazi2026blinkthink,
231
+ title={BlinkThink: Training-Free Calibrated Dual-Process Routing for Discrete LLM Decisions},
232
+ author={Petnehazi, Gabor},
233
+ year={2026},
234
+ url={https://github.com/RebuildAI-UK/blinkthink}
235
+ }
236
+ ```
237
+
238
+ ---
239
+
240
+ ## License
241
+
242
+ MIT License. See [LICENSE](LICENSE) for details.
@@ -0,0 +1,184 @@
1
+ # BlinkThink
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/blinkthink.svg)](https://pypi.org/project/blinkthink/)
4
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
5
+ [![Python](https://img.shields.io/badge/Python-3.10%2B-blue)](https://www.python.org/)
6
+ [![Pydantic](https://img.shields.io/badge/Pydantic-v2-green)](https://pydantic.dev/)
7
+
8
+ **Training-Free Calibrated Dual-Process Routing for Open-Weight Language Models.**
9
+
10
+ BlinkThink implements Daniel Kahneman's **System 1 / System 2** cognitive paradigm for discrete LLM decisions (classification, triage, assertion verification, and multiple-choice QA).
11
+
12
+ - **System 1 (`BlinkEngine`):** Single batched prefill pass with $N$ cyclic option permutations (`0` autoregressively decoded tokens, ~130–240ms on 4B–12B NF4 models).
13
+ - **System 2 (`ThinkEngine`):** Deliberative Chain-of-Thought (CoT) with XML tag delimitation and early exit, triggered selectively only when System 1 confidence falls below a validation-tuned routing threshold $\tau^*$.
14
+
15
+ ---
16
+
17
+ ## Architecture Overview
18
+
19
+ ```
20
+ ┌────────────────────────────┐
21
+ │ Incoming Decision │
22
+ │ (Choice / Binary) │
23
+ └─────────────┬──────────────┘
24
+ │
25
+ ▼
26
+ ┌────────────────────────────┐
27
+ │ System 1: BlinkEngine │
28
+ │ • N cyclic permutations │
29
+ │ • Single batched prefill │
30
+ │ • Temperature scaling T* │
31
+ └─────────────┬──────────────┘
32
+ │
33
+ Confidence >= tau* ?
34
+ │
35
+ ┌──────────────┴──────────────┐
36
+ Yes │ │ No
37
+ (Fast Path) ▼ ▼ (Deliberate Path)
38
+ ┌─────────────────────────┐ ┌───────────────────────────┐
39
+ │ Instant Return │ │ System 2: ThinkEngine │
40
+ │ • 0 decoded tokens │ │ • Autoregressive CoT │
41
+ │ • Sub-second prefill │ │ • StopStringCriteria │
42
+ │ • Calibrated probs │ │ • Calibrated token probs │
43
+ └─────────────────────────┘ └─────────────┬─────────────┘
44
+ │
45
+ ▼
46
+ ┌───────────────────────────┐
47
+ │ Final Decision & Trace │
48
+ └───────────────────────────┘
49
+ ```
50
+
51
+ ---
52
+
53
+ ## Installation
54
+
55
+ Install from PyPI:
56
+
57
+ ```bash
58
+ # Core package (local inference & remote HTTP client)
59
+ pip install blinkthink
60
+
61
+ # With 4-bit/8-bit quantization support
62
+ pip install "blinkthink[quantization]"
63
+
64
+ # With FastAPI server, benchmarks, and quantization
65
+ pip install "blinkthink[all]"
66
+ ```
67
+
68
+ Or install from source for development:
69
+
70
+ ```bash
71
+ git clone https://github.com/RebuildAI-UK/blinkthink.git
72
+ cd blinkthink
73
+ pip install -e ".[all,dev]"
74
+ ```
75
+
76
+ ---
77
+
78
+ ## Quick Example
79
+
80
+ ```python
81
+ import torch
82
+ from transformers import AutoModelForCausalLM, AutoTokenizer
83
+ from blinkthink import BlinkThink
84
+
85
+ model_id = "google/gemma-4-12B-it"
86
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
87
+ model = AutoModelForCausalLM.from_pretrained(
88
+ model_id,
89
+ torch_dtype=torch.bfloat16,
90
+ device_map="auto"
91
+ )
92
+
93
+ client = BlinkThink(model=model, tokenizer=tokenizer, threshold=0.35, n_blinks=2)
94
+
95
+ # 1. Multi-Choice Classification (with optional per-call threshold / blink overrides)
96
+ res = client.choice(
97
+ prompt="Which organelle is responsible for cellular ATP synthesis?",
98
+ options=["Golgi Apparatus", "Mitochondria", "Ribosome", "Endoplasmic Reticulum"],
99
+ n_blinks=2, # Override blink count per-decision
100
+ threshold=0.35, # Override escalation cutoff per-decision
101
+ )
102
+ print(f"Decision: {res.decision}")
103
+ print(f"Confidence: {res.confidence:.3f}")
104
+ print(f"System: {res.system_used} ({res.latency_ms:.1f}ms)")
105
+
106
+ # 2. Binary Assertion Verification
107
+ guard = client.binary(
108
+ prompt="Is light faster in glass than in vacuum?",
109
+ threshold=0.40,
110
+ )
111
+ print(f"Decision: {guard.decision}") # "No"
112
+ ```
113
+
114
+ ---
115
+
116
+ ## Benchmark Highlights
117
+
118
+ Evaluated under 4-bit NF4 quantization on MMLU ($K=4$, $M_{\text{test}}=1000$) and BoolQ ($K=2$, $M_{\text{test}}=800$):
119
+
120
+ | Model | Dataset | Mode | Acc (%) | Latency | Sys 2 Esc (%) |
121
+ | :--- | :--- | :--- | :---: | :---: | :---: |
122
+ | `google/gemma-4-12B-it` | MMLU | CoT Baseline | 79.4 | 57.80 s | 100.0 |
123
+ | `google/gemma-4-12B-it` | MMLU | **BlinkThink ($N=2$, Balanced)** | **78.9** | **19.16 s** (-66.8%) | **28.9** |
124
+ | `google/gemma-4-12B-it` | BoolQ | CoT Baseline | 89.9 | 27.45 s | 100.0 |
125
+ | `google/gemma-4-12B-it` | BoolQ | **BlinkThink ($N=2$, Balanced)** | **89.5** | **3.94 s** (-85.6%) | **8.8** |
126
+ | `Qwen/Qwen3.5-4B` | MMLU | CoT Baseline | 43.7 | 46.15 s | 100.0 |
127
+ | `Qwen/Qwen3.5-4B` | MMLU | **BlinkThink ($N=4$, Cyclic)** | **68.4** | **190.4 ms** | **0.0** |
128
+ | `Qwen/Qwen3.5-4B` | BoolQ | CoT Baseline | 86.4 | 44.48 s | 100.0 |
129
+ | `Qwen/Qwen3.5-4B` | BoolQ | **BlinkThink ($N=2$, Pure S1)** | **86.1** | **149.1 ms** | **0.0** |
130
+
131
+ ---
132
+
133
+ ## Guides & Documentation
134
+
135
+ - **[Quickstart Guide](docs/QUICKSTART.md):** Setup, local usage, and production REST API deployment.
136
+ - **[Decision API Guide](docs/DECISION_API_GUIDE.md):** Mathematical formulas, permutation invariance, temperature calibration, and optimal $\tau^*$ routing theory.
137
+ - **[Examples Directory](examples/):**
138
+ - `01_in_process_sdk.py`: In-memory GPU SDK usage.
139
+ - `02_fastapi_server.py`: FastAPI OpenAPI REST deployment.
140
+ - `03_remote_client.py`: Sync & Async HTTP client usage.
141
+
142
+ ---
143
+
144
+ ## Running Benchmarks
145
+
146
+ Run the built-in study harness across curated or standard Hugging Face datasets:
147
+
148
+ ```bash
149
+ # 1. Single study run (fast curated sanity test)
150
+ python benchmarks/run_study.py --dataset curated
151
+
152
+ # 2. Single study run on MMLU
153
+ python benchmarks/run_study.py --dataset mmlu --val_size 50 --test_size 200 --tune_objective balanced
154
+
155
+ # 3. Run complete matrix across default models and all objectives
156
+ python benchmarks/run_study_matrix.py --run --dataset curated
157
+
158
+ # 4. Run matrix across custom models on MMLU
159
+ python benchmarks/run_study_matrix.py --run --models Qwen/Qwen3.5-4B google/gemma-4-12B-it --dataset mmlu
160
+
161
+ # 5. Inspect and compare existing results (auto-detects datasets)
162
+ python benchmarks/run_study_matrix.py
163
+ ```
164
+
165
+ ---
166
+
167
+ ## Citation
168
+
169
+ If you use BlinkThink in your research, please cite:
170
+
171
+ ```bibtex
172
+ @article{petnehazi2026blinkthink,
173
+ title={BlinkThink: Training-Free Calibrated Dual-Process Routing for Discrete LLM Decisions},
174
+ author={Petnehazi, Gabor},
175
+ year={2026},
176
+ url={https://github.com/RebuildAI-UK/blinkthink}
177
+ }
178
+ ```
179
+
180
+ ---
181
+
182
+ ## License
183
+
184
+ MIT License. See [LICENSE](LICENSE) for details.
@@ -0,0 +1,39 @@
1
+ """
2
+ BlinkThink: Calibrated Dual-Process Routing for Open-Weight Language Models.
3
+ """
4
+
5
+ from .benchmark import BenchmarkReport, BlinkStudyEvaluator, compute_ece
6
+ from .calibrator import TemperatureCalibrator
7
+ from .client import BlinkThink
8
+ from .remote import AsyncBlinkThinkRemoteClient, BlinkThinkRemoteClient
9
+ from .router import BlinkThinkRouter
10
+ from .system1 import BlinkEngine
11
+ from .system2 import ThinkEngine
12
+ from .tuner import BlinkThinkTuner, TuningReport
13
+ from .schemas import (
14
+ BlinkTelemetry,
15
+ DecisionRequest,
16
+ DecisionResult,
17
+ DecisionType,
18
+ )
19
+
20
+ __version__ = "1.1.0"
21
+
22
+ __all__ = [
23
+ "BlinkThink",
24
+ "BlinkThinkRemoteClient",
25
+ "AsyncBlinkThinkRemoteClient",
26
+ "BlinkThinkRouter",
27
+ "BlinkEngine",
28
+ "ThinkEngine",
29
+ "BlinkThinkTuner",
30
+ "TemperatureCalibrator",
31
+ "BlinkStudyEvaluator",
32
+ "DecisionRequest",
33
+ "DecisionResult",
34
+ "DecisionType",
35
+ "BlinkTelemetry",
36
+ "BenchmarkReport",
37
+ "TuningReport",
38
+ "compute_ece",
39
+ ]
@@ -0,0 +1,103 @@
1
+ from dataclasses import asdict, dataclass
2
+ from typing import Any, Dict, List, Optional, Tuple
3
+ import numpy as np
4
+ from .router import BlinkThinkRouter
5
+ from .schemas import DecisionRequest
6
+
7
+
8
+ @dataclass
9
+ class BenchmarkReport:
10
+ config_name: str
11
+ n_blinks: int
12
+ threshold: float
13
+ accuracy: float
14
+ avg_latency_ms: float
15
+ system2_escalation_pct: float
16
+ expected_calibration_error: float
17
+ avg_drift: float
18
+ avg_consensus: float
19
+
20
+ def to_dict(self) -> Dict[str, Any]:
21
+ return asdict(self)
22
+
23
+
24
+ def compute_ece(probs: np.ndarray, labels: np.ndarray, n_bins: int = 10) -> float:
25
+ """
26
+ Computes Expected Calibration Error (ECE).
27
+ """
28
+ if len(probs) == 0:
29
+ return 0.0
30
+ probs = np.clip(probs, 0.0, 1.0)
31
+ bin_boundaries = np.linspace(0, 1, n_bins + 1)
32
+ ece = 0.0
33
+ for i in range(n_bins):
34
+ bin_lower, bin_upper = bin_boundaries[i], bin_boundaries[i + 1]
35
+ in_bin = (
36
+ (probs >= bin_lower) & (probs <= bin_upper)
37
+ if i == 0
38
+ else (probs > bin_lower) & (probs <= bin_upper)
39
+ )
40
+ prop_in_bin = np.mean(in_bin)
41
+ if prop_in_bin > 0:
42
+ accuracy_in_bin = np.mean(labels[in_bin])
43
+ avg_confidence_in_bin = np.mean(probs[in_bin])
44
+ ece += np.abs(accuracy_in_bin - avg_confidence_in_bin) * prop_in_bin
45
+ return float(ece)
46
+
47
+
48
+ class BlinkStudyEvaluator:
49
+ def __init__(self, router: BlinkThinkRouter):
50
+ self.router = router
51
+
52
+ def evaluate_dataset(
53
+ self,
54
+ dataset: List[Tuple[DecisionRequest, Any]],
55
+ config_name: str = "BlinkThink",
56
+ n_blinks: int = 2,
57
+ threshold: float = 0.35,
58
+ force_mode: Optional[str] = None,
59
+ ) -> BenchmarkReport:
60
+ correct: List[int] = []
61
+ latencies: List[float] = []
62
+ escalated: List[int] = []
63
+ confidences: List[float] = []
64
+ drifts: List[float] = []
65
+ consensuses: List[float] = []
66
+
67
+ for req, y_true in dataset:
68
+ res = self.router.decide(
69
+ req,
70
+ n_blinks=n_blinks,
71
+ threshold=threshold,
72
+ force_mode=force_mode,
73
+ )
74
+ is_correct = (res.decision == y_true)
75
+
76
+ correct.append(1 if is_correct else 0)
77
+ latencies.append(res.latency_ms)
78
+ escalated.append(1 if res.system_used in ("system_2", "system_1_fallback") else 0)
79
+
80
+ top_prob = (
81
+ res.probabilities.get(res.decision, res.confidence)
82
+ if res.probabilities
83
+ else res.confidence
84
+ )
85
+ confidences.append(top_prob)
86
+
87
+ if res.telemetry:
88
+ drifts.append(res.telemetry.drift)
89
+ consensuses.append(res.telemetry.consensus_rate)
90
+
91
+ ece = compute_ece(np.array(confidences), np.array(correct))
92
+
93
+ return BenchmarkReport(
94
+ config_name=config_name,
95
+ n_blinks=n_blinks,
96
+ threshold=threshold,
97
+ accuracy=float(np.mean(correct)) if correct else 0.0,
98
+ avg_latency_ms=float(np.mean(latencies)) if latencies else 0.0,
99
+ system2_escalation_pct=float(np.mean(escalated) * 100.0) if escalated else 0.0,
100
+ expected_calibration_error=ece,
101
+ avg_drift=float(np.mean(drifts)) if drifts else 0.0,
102
+ avg_consensus=float(np.mean(consensuses)) if consensuses else 1.0,
103
+ )
@@ -0,0 +1,89 @@
1
+ import logging
2
+ from typing import Any, Dict, Optional, Tuple
3
+ from scipy.optimize import minimize_scalar
4
+ import torch
5
+ import torch.nn.functional as F
6
+ from .schemas import DecisionType
7
+
8
+ logger = logging.getLogger(__name__)
9
+
10
+
11
+ class TemperatureCalibrator:
12
+ """
13
+ Manages and optimizes temperature scaling for choice decisions.
14
+ """
15
+ def __init__(
16
+ self,
17
+ temperature: float = 3.4,
18
+ temperature_choice: Optional[float] = None,
19
+ enable_prior_correction: bool = False,
20
+ **kwargs: Any,
21
+ ):
22
+ temp = temperature_choice if temperature_choice is not None else temperature
23
+ self.temperature = float(temp)
24
+ self.enable_prior_correction = enable_prior_correction
25
+
26
+ @property
27
+ def temperatures(self) -> Dict[DecisionType, float]:
28
+ return {DecisionType.CHOICE: self.temperature}
29
+
30
+ def calibrate(self, logits: torch.Tensor, decision_type: Optional[DecisionType] = None) -> torch.Tensor:
31
+ temp = max(self.temperature, 1e-4)
32
+ scaled = logits / temp
33
+ return F.softmax(scaled, dim=-1)
34
+
35
+ def fit(
36
+ self,
37
+ candidate_logits: Any,
38
+ target_indices: Any,
39
+ decision_type: Optional[DecisionType] = None,
40
+ temp_bounds: Tuple[float, float] = (0.1, 10.0),
41
+ ) -> float:
42
+ """
43
+ Finds optimal temperature T* by minimizing Negative Log-Likelihood (NLL).
44
+ """
45
+ if isinstance(candidate_logits, torch.Tensor) and candidate_logits.ndim == 2:
46
+ logits_list = [row for row in candidate_logits.detach().cpu().float()]
47
+ else:
48
+ logits_list = [
49
+ row.detach().cpu().float() if isinstance(row, torch.Tensor)
50
+ else torch.tensor(row, dtype=torch.float32)
51
+ for row in candidate_logits
52
+ ]
53
+
54
+ targets_list = (
55
+ target_indices.detach().cpu().long().tolist()
56
+ if isinstance(target_indices, torch.Tensor)
57
+ else [int(t) for t in target_indices]
58
+ )
59
+
60
+ if len(logits_list) < 3:
61
+ logger.warning(
62
+ "Only %d samples provided for calibration. Retaining temperature T=%.2f.",
63
+ len(logits_list),
64
+ self.temperature,
65
+ )
66
+ return self.temperature
67
+
68
+ def nll_objective(t: float) -> float:
69
+ t_clamped = max(t, 1e-4)
70
+ total_nll = 0.0
71
+ for l_row, target in zip(logits_list, targets_list):
72
+ scaled = l_row / t_clamped
73
+ log_probs = F.log_softmax(scaled, dim=-1)
74
+ total_nll += -log_probs[target].item()
75
+ return total_nll / len(logits_list)
76
+
77
+ res = minimize_scalar(nll_objective, bounds=temp_bounds, method="bounded")
78
+ self.temperature = float(res.x)
79
+ logger.info("Temperature calibrated: T* = %.3f", self.temperature)
80
+ return self.temperature
81
+
82
+ def to_dict(self) -> Dict[str, float]:
83
+ return {"choice": self.temperature}
84
+
85
+ def load_dict(self, data: Dict[str, float]):
86
+ if "choice" in data:
87
+ self.temperature = float(data["choice"])
88
+ elif "temperature" in data:
89
+ self.temperature = float(data["temperature"])