blinkthink 1.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- blinkthink/__init__.py +39 -0
- blinkthink/benchmark.py +103 -0
- blinkthink/calibrator.py +89 -0
- blinkthink/client.py +126 -0
- blinkthink/py.typed +0 -0
- blinkthink/remote.py +181 -0
- blinkthink/router.py +110 -0
- blinkthink/schemas.py +125 -0
- blinkthink/server.py +197 -0
- blinkthink/system1.py +225 -0
- blinkthink/system2.py +408 -0
- blinkthink/tuner.py +185 -0
- blinkthink-1.1.0.dist-info/METADATA +242 -0
- blinkthink-1.1.0.dist-info/RECORD +18 -0
- blinkthink-1.1.0.dist-info/WHEEL +5 -0
- blinkthink-1.1.0.dist-info/entry_points.txt +2 -0
- blinkthink-1.1.0.dist-info/licenses/LICENSE +21 -0
- blinkthink-1.1.0.dist-info/top_level.txt +1 -0
blinkthink/__init__.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
BlinkThink: Calibrated Dual-Process Routing for Open-Weight Language Models.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .benchmark import BenchmarkReport, BlinkStudyEvaluator, compute_ece
|
|
6
|
+
from .calibrator import TemperatureCalibrator
|
|
7
|
+
from .client import BlinkThink
|
|
8
|
+
from .remote import AsyncBlinkThinkRemoteClient, BlinkThinkRemoteClient
|
|
9
|
+
from .router import BlinkThinkRouter
|
|
10
|
+
from .system1 import BlinkEngine
|
|
11
|
+
from .system2 import ThinkEngine
|
|
12
|
+
from .tuner import BlinkThinkTuner, TuningReport
|
|
13
|
+
from .schemas import (
|
|
14
|
+
BlinkTelemetry,
|
|
15
|
+
DecisionRequest,
|
|
16
|
+
DecisionResult,
|
|
17
|
+
DecisionType,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
__version__ = "1.1.0"
|
|
21
|
+
|
|
22
|
+
__all__ = [
|
|
23
|
+
"BlinkThink",
|
|
24
|
+
"BlinkThinkRemoteClient",
|
|
25
|
+
"AsyncBlinkThinkRemoteClient",
|
|
26
|
+
"BlinkThinkRouter",
|
|
27
|
+
"BlinkEngine",
|
|
28
|
+
"ThinkEngine",
|
|
29
|
+
"BlinkThinkTuner",
|
|
30
|
+
"TemperatureCalibrator",
|
|
31
|
+
"BlinkStudyEvaluator",
|
|
32
|
+
"DecisionRequest",
|
|
33
|
+
"DecisionResult",
|
|
34
|
+
"DecisionType",
|
|
35
|
+
"BlinkTelemetry",
|
|
36
|
+
"BenchmarkReport",
|
|
37
|
+
"TuningReport",
|
|
38
|
+
"compute_ece",
|
|
39
|
+
]
|
blinkthink/benchmark.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from dataclasses import asdict, dataclass
|
|
2
|
+
from typing import Any, Dict, List, Optional, Tuple
|
|
3
|
+
import numpy as np
|
|
4
|
+
from .router import BlinkThinkRouter
|
|
5
|
+
from .schemas import DecisionRequest
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class BenchmarkReport:
|
|
10
|
+
config_name: str
|
|
11
|
+
n_blinks: int
|
|
12
|
+
threshold: float
|
|
13
|
+
accuracy: float
|
|
14
|
+
avg_latency_ms: float
|
|
15
|
+
system2_escalation_pct: float
|
|
16
|
+
expected_calibration_error: float
|
|
17
|
+
avg_drift: float
|
|
18
|
+
avg_consensus: float
|
|
19
|
+
|
|
20
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
21
|
+
return asdict(self)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def compute_ece(probs: np.ndarray, labels: np.ndarray, n_bins: int = 10) -> float:
|
|
25
|
+
"""
|
|
26
|
+
Computes Expected Calibration Error (ECE).
|
|
27
|
+
"""
|
|
28
|
+
if len(probs) == 0:
|
|
29
|
+
return 0.0
|
|
30
|
+
probs = np.clip(probs, 0.0, 1.0)
|
|
31
|
+
bin_boundaries = np.linspace(0, 1, n_bins + 1)
|
|
32
|
+
ece = 0.0
|
|
33
|
+
for i in range(n_bins):
|
|
34
|
+
bin_lower, bin_upper = bin_boundaries[i], bin_boundaries[i + 1]
|
|
35
|
+
in_bin = (
|
|
36
|
+
(probs >= bin_lower) & (probs <= bin_upper)
|
|
37
|
+
if i == 0
|
|
38
|
+
else (probs > bin_lower) & (probs <= bin_upper)
|
|
39
|
+
)
|
|
40
|
+
prop_in_bin = np.mean(in_bin)
|
|
41
|
+
if prop_in_bin > 0:
|
|
42
|
+
accuracy_in_bin = np.mean(labels[in_bin])
|
|
43
|
+
avg_confidence_in_bin = np.mean(probs[in_bin])
|
|
44
|
+
ece += np.abs(accuracy_in_bin - avg_confidence_in_bin) * prop_in_bin
|
|
45
|
+
return float(ece)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class BlinkStudyEvaluator:
|
|
49
|
+
def __init__(self, router: BlinkThinkRouter):
|
|
50
|
+
self.router = router
|
|
51
|
+
|
|
52
|
+
def evaluate_dataset(
|
|
53
|
+
self,
|
|
54
|
+
dataset: List[Tuple[DecisionRequest, Any]],
|
|
55
|
+
config_name: str = "BlinkThink",
|
|
56
|
+
n_blinks: int = 2,
|
|
57
|
+
threshold: float = 0.35,
|
|
58
|
+
force_mode: Optional[str] = None,
|
|
59
|
+
) -> BenchmarkReport:
|
|
60
|
+
correct: List[int] = []
|
|
61
|
+
latencies: List[float] = []
|
|
62
|
+
escalated: List[int] = []
|
|
63
|
+
confidences: List[float] = []
|
|
64
|
+
drifts: List[float] = []
|
|
65
|
+
consensuses: List[float] = []
|
|
66
|
+
|
|
67
|
+
for req, y_true in dataset:
|
|
68
|
+
res = self.router.decide(
|
|
69
|
+
req,
|
|
70
|
+
n_blinks=n_blinks,
|
|
71
|
+
threshold=threshold,
|
|
72
|
+
force_mode=force_mode,
|
|
73
|
+
)
|
|
74
|
+
is_correct = (res.decision == y_true)
|
|
75
|
+
|
|
76
|
+
correct.append(1 if is_correct else 0)
|
|
77
|
+
latencies.append(res.latency_ms)
|
|
78
|
+
escalated.append(1 if res.system_used in ("system_2", "system_1_fallback") else 0)
|
|
79
|
+
|
|
80
|
+
top_prob = (
|
|
81
|
+
res.probabilities.get(res.decision, res.confidence)
|
|
82
|
+
if res.probabilities
|
|
83
|
+
else res.confidence
|
|
84
|
+
)
|
|
85
|
+
confidences.append(top_prob)
|
|
86
|
+
|
|
87
|
+
if res.telemetry:
|
|
88
|
+
drifts.append(res.telemetry.drift)
|
|
89
|
+
consensuses.append(res.telemetry.consensus_rate)
|
|
90
|
+
|
|
91
|
+
ece = compute_ece(np.array(confidences), np.array(correct))
|
|
92
|
+
|
|
93
|
+
return BenchmarkReport(
|
|
94
|
+
config_name=config_name,
|
|
95
|
+
n_blinks=n_blinks,
|
|
96
|
+
threshold=threshold,
|
|
97
|
+
accuracy=float(np.mean(correct)) if correct else 0.0,
|
|
98
|
+
avg_latency_ms=float(np.mean(latencies)) if latencies else 0.0,
|
|
99
|
+
system2_escalation_pct=float(np.mean(escalated) * 100.0) if escalated else 0.0,
|
|
100
|
+
expected_calibration_error=ece,
|
|
101
|
+
avg_drift=float(np.mean(drifts)) if drifts else 0.0,
|
|
102
|
+
avg_consensus=float(np.mean(consensuses)) if consensuses else 1.0,
|
|
103
|
+
)
|
blinkthink/calibrator.py
ADDED
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Any, Dict, Optional, Tuple
|
|
3
|
+
from scipy.optimize import minimize_scalar
|
|
4
|
+
import torch
|
|
5
|
+
import torch.nn.functional as F
|
|
6
|
+
from .schemas import DecisionType
|
|
7
|
+
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TemperatureCalibrator:
|
|
12
|
+
"""
|
|
13
|
+
Manages and optimizes temperature scaling for choice decisions.
|
|
14
|
+
"""
|
|
15
|
+
def __init__(
|
|
16
|
+
self,
|
|
17
|
+
temperature: float = 3.4,
|
|
18
|
+
temperature_choice: Optional[float] = None,
|
|
19
|
+
enable_prior_correction: bool = False,
|
|
20
|
+
**kwargs: Any,
|
|
21
|
+
):
|
|
22
|
+
temp = temperature_choice if temperature_choice is not None else temperature
|
|
23
|
+
self.temperature = float(temp)
|
|
24
|
+
self.enable_prior_correction = enable_prior_correction
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def temperatures(self) -> Dict[DecisionType, float]:
|
|
28
|
+
return {DecisionType.CHOICE: self.temperature}
|
|
29
|
+
|
|
30
|
+
def calibrate(self, logits: torch.Tensor, decision_type: Optional[DecisionType] = None) -> torch.Tensor:
|
|
31
|
+
temp = max(self.temperature, 1e-4)
|
|
32
|
+
scaled = logits / temp
|
|
33
|
+
return F.softmax(scaled, dim=-1)
|
|
34
|
+
|
|
35
|
+
def fit(
|
|
36
|
+
self,
|
|
37
|
+
candidate_logits: Any,
|
|
38
|
+
target_indices: Any,
|
|
39
|
+
decision_type: Optional[DecisionType] = None,
|
|
40
|
+
temp_bounds: Tuple[float, float] = (0.1, 10.0),
|
|
41
|
+
) -> float:
|
|
42
|
+
"""
|
|
43
|
+
Finds optimal temperature T* by minimizing Negative Log-Likelihood (NLL).
|
|
44
|
+
"""
|
|
45
|
+
if isinstance(candidate_logits, torch.Tensor) and candidate_logits.ndim == 2:
|
|
46
|
+
logits_list = [row for row in candidate_logits.detach().cpu().float()]
|
|
47
|
+
else:
|
|
48
|
+
logits_list = [
|
|
49
|
+
row.detach().cpu().float() if isinstance(row, torch.Tensor)
|
|
50
|
+
else torch.tensor(row, dtype=torch.float32)
|
|
51
|
+
for row in candidate_logits
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
targets_list = (
|
|
55
|
+
target_indices.detach().cpu().long().tolist()
|
|
56
|
+
if isinstance(target_indices, torch.Tensor)
|
|
57
|
+
else [int(t) for t in target_indices]
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if len(logits_list) < 3:
|
|
61
|
+
logger.warning(
|
|
62
|
+
"Only %d samples provided for calibration. Retaining temperature T=%.2f.",
|
|
63
|
+
len(logits_list),
|
|
64
|
+
self.temperature,
|
|
65
|
+
)
|
|
66
|
+
return self.temperature
|
|
67
|
+
|
|
68
|
+
def nll_objective(t: float) -> float:
|
|
69
|
+
t_clamped = max(t, 1e-4)
|
|
70
|
+
total_nll = 0.0
|
|
71
|
+
for l_row, target in zip(logits_list, targets_list):
|
|
72
|
+
scaled = l_row / t_clamped
|
|
73
|
+
log_probs = F.log_softmax(scaled, dim=-1)
|
|
74
|
+
total_nll += -log_probs[target].item()
|
|
75
|
+
return total_nll / len(logits_list)
|
|
76
|
+
|
|
77
|
+
res = minimize_scalar(nll_objective, bounds=temp_bounds, method="bounded")
|
|
78
|
+
self.temperature = float(res.x)
|
|
79
|
+
logger.info("Temperature calibrated: T* = %.3f", self.temperature)
|
|
80
|
+
return self.temperature
|
|
81
|
+
|
|
82
|
+
def to_dict(self) -> Dict[str, float]:
|
|
83
|
+
return {"choice": self.temperature}
|
|
84
|
+
|
|
85
|
+
def load_dict(self, data: Dict[str, float]):
|
|
86
|
+
if "choice" in data:
|
|
87
|
+
self.temperature = float(data["choice"])
|
|
88
|
+
elif "temperature" in data:
|
|
89
|
+
self.temperature = float(data["temperature"])
|
blinkthink/client.py
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""
|
|
2
|
+
BlinkThink Developer Client: Pythonic ergonomic API for local and embedded execution.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
7
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
|
8
|
+
|
|
9
|
+
import torch
|
|
10
|
+
from .calibrator import TemperatureCalibrator
|
|
11
|
+
from .router import BlinkThinkRouter
|
|
12
|
+
from .schemas import DecisionRequest, DecisionResult, DecisionType
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class BlinkThink:
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
model: Any,
|
|
19
|
+
tokenizer: Any,
|
|
20
|
+
threshold: float = 0.35,
|
|
21
|
+
n_blinks: int = 2,
|
|
22
|
+
strict_consensus: bool = False,
|
|
23
|
+
calibrator: Optional[TemperatureCalibrator] = None,
|
|
24
|
+
max_think_tokens: int = 512,
|
|
25
|
+
device: Optional[Union[str, torch.device]] = None,
|
|
26
|
+
config_path: Optional[str] = None,
|
|
27
|
+
prompt_mode: Literal["auto", "chat", "completion"] = "auto",
|
|
28
|
+
confidence_metric: Literal["composite", "margin", "msp"] = "composite",
|
|
29
|
+
):
|
|
30
|
+
self.router = BlinkThinkRouter(
|
|
31
|
+
model=model,
|
|
32
|
+
tokenizer=tokenizer,
|
|
33
|
+
n_blinks=n_blinks,
|
|
34
|
+
threshold=threshold,
|
|
35
|
+
strict_consensus=strict_consensus,
|
|
36
|
+
calibrator=calibrator,
|
|
37
|
+
max_think_tokens=max_think_tokens,
|
|
38
|
+
device=device,
|
|
39
|
+
prompt_mode=prompt_mode,
|
|
40
|
+
confidence_metric=confidence_metric,
|
|
41
|
+
)
|
|
42
|
+
if config_path:
|
|
43
|
+
self.router.load_config(config_path)
|
|
44
|
+
|
|
45
|
+
self._executor = ThreadPoolExecutor(max_workers=1)
|
|
46
|
+
|
|
47
|
+
def choice(
|
|
48
|
+
self,
|
|
49
|
+
prompt: str,
|
|
50
|
+
options: List[str],
|
|
51
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
52
|
+
n_blinks: Optional[int] = None,
|
|
53
|
+
threshold: Optional[float] = None,
|
|
54
|
+
force_mode: Optional[Literal["blink_only", "deliberate", "force_deliberate", "think_only"]] = None,
|
|
55
|
+
) -> DecisionResult[str]:
|
|
56
|
+
req = DecisionRequest(
|
|
57
|
+
prompt=prompt,
|
|
58
|
+
decision_type=DecisionType.CHOICE,
|
|
59
|
+
options=options,
|
|
60
|
+
metadata=metadata,
|
|
61
|
+
n_blinks=n_blinks,
|
|
62
|
+
threshold=threshold,
|
|
63
|
+
force_mode=force_mode,
|
|
64
|
+
)
|
|
65
|
+
return self.decide(req, n_blinks=n_blinks, threshold=threshold, force_mode=force_mode)
|
|
66
|
+
|
|
67
|
+
def binary(
|
|
68
|
+
self,
|
|
69
|
+
prompt: str,
|
|
70
|
+
options: Optional[List[str]] = None,
|
|
71
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
72
|
+
n_blinks: Optional[int] = None,
|
|
73
|
+
threshold: Optional[float] = None,
|
|
74
|
+
force_mode: Optional[Literal["blink_only", "deliberate", "force_deliberate", "think_only"]] = None,
|
|
75
|
+
) -> DecisionResult[str]:
|
|
76
|
+
return self.choice(
|
|
77
|
+
prompt=prompt,
|
|
78
|
+
options=options or ["Yes", "No"],
|
|
79
|
+
metadata=metadata,
|
|
80
|
+
n_blinks=n_blinks,
|
|
81
|
+
threshold=threshold,
|
|
82
|
+
force_mode=force_mode,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
def decide(
|
|
86
|
+
self,
|
|
87
|
+
request: Union[DecisionRequest, Dict[str, Any]],
|
|
88
|
+
n_blinks: Optional[int] = None,
|
|
89
|
+
threshold: Optional[float] = None,
|
|
90
|
+
force_mode: Optional[Literal["blink_only", "deliberate", "force_deliberate", "system_1", "system_2", "think_only"]] = None,
|
|
91
|
+
) -> DecisionResult[str]:
|
|
92
|
+
if isinstance(request, dict):
|
|
93
|
+
request = DecisionRequest(**request)
|
|
94
|
+
return self.router.decide(
|
|
95
|
+
request=request,
|
|
96
|
+
n_blinks=n_blinks,
|
|
97
|
+
threshold=threshold,
|
|
98
|
+
force_mode=force_mode,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
async def adecide(self, *args, **kwargs) -> DecisionResult[str]:
|
|
102
|
+
loop = asyncio.get_running_loop()
|
|
103
|
+
return await loop.run_in_executor(self._executor, lambda: self.decide(*args, **kwargs))
|
|
104
|
+
|
|
105
|
+
async def achoice(self, *args, **kwargs) -> DecisionResult[str]:
|
|
106
|
+
loop = asyncio.get_running_loop()
|
|
107
|
+
return await loop.run_in_executor(self._executor, lambda: self.choice(*args, **kwargs))
|
|
108
|
+
|
|
109
|
+
async def abinary(self, *args, **kwargs) -> DecisionResult[str]:
|
|
110
|
+
loop = asyncio.get_running_loop()
|
|
111
|
+
return await loop.run_in_executor(self._executor, lambda: self.binary(*args, **kwargs))
|
|
112
|
+
|
|
113
|
+
def save_config(self, filepath: str):
|
|
114
|
+
self.router.save_config(filepath)
|
|
115
|
+
|
|
116
|
+
def load_config(self, filepath: str):
|
|
117
|
+
self.router.load_config(filepath)
|
|
118
|
+
|
|
119
|
+
def close(self):
|
|
120
|
+
self._executor.shutdown(wait=False)
|
|
121
|
+
|
|
122
|
+
def __enter__(self):
|
|
123
|
+
return self
|
|
124
|
+
|
|
125
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
126
|
+
self.close()
|
blinkthink/py.typed
ADDED
|
File without changes
|
blinkthink/remote.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""
|
|
2
|
+
BlinkThink Remote Client: Light-weight HTTP client for remote service interaction.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
|
6
|
+
import httpx
|
|
7
|
+
from .schemas import DecisionRequest, DecisionResult, DecisionType
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class BlinkThinkRemoteClient:
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
base_url: str = "http://localhost:8000",
|
|
14
|
+
api_key: Optional[str] = None,
|
|
15
|
+
timeout: float = 60.0,
|
|
16
|
+
):
|
|
17
|
+
self.base_url = base_url.rstrip("/")
|
|
18
|
+
headers = {"Content-Type": "application/json"}
|
|
19
|
+
if api_key:
|
|
20
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
21
|
+
self.client = httpx.Client(base_url=self.base_url, headers=headers, timeout=timeout)
|
|
22
|
+
|
|
23
|
+
def __enter__(self):
|
|
24
|
+
return self
|
|
25
|
+
|
|
26
|
+
def __exit__(self, *args):
|
|
27
|
+
self.close()
|
|
28
|
+
|
|
29
|
+
def close(self):
|
|
30
|
+
self.client.close()
|
|
31
|
+
|
|
32
|
+
def choice(
|
|
33
|
+
self,
|
|
34
|
+
prompt: str,
|
|
35
|
+
options: List[str],
|
|
36
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
37
|
+
n_blinks: Optional[int] = None,
|
|
38
|
+
threshold: Optional[float] = None,
|
|
39
|
+
force_mode: Optional[Literal["blink_only", "deliberate", "force_deliberate", "think_only"]] = None,
|
|
40
|
+
) -> DecisionResult[str]:
|
|
41
|
+
req = DecisionRequest(
|
|
42
|
+
prompt=prompt,
|
|
43
|
+
decision_type=DecisionType.CHOICE,
|
|
44
|
+
options=options,
|
|
45
|
+
metadata=metadata,
|
|
46
|
+
n_blinks=n_blinks,
|
|
47
|
+
threshold=threshold,
|
|
48
|
+
force_mode=force_mode,
|
|
49
|
+
)
|
|
50
|
+
return self.decide(req, n_blinks=n_blinks, threshold=threshold, force_mode=force_mode)
|
|
51
|
+
|
|
52
|
+
def binary(
|
|
53
|
+
self,
|
|
54
|
+
prompt: str,
|
|
55
|
+
options: Optional[List[str]] = None,
|
|
56
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
57
|
+
n_blinks: Optional[int] = None,
|
|
58
|
+
threshold: Optional[float] = None,
|
|
59
|
+
force_mode: Optional[Literal["blink_only", "deliberate", "force_deliberate", "think_only"]] = None,
|
|
60
|
+
) -> DecisionResult[str]:
|
|
61
|
+
return self.choice(
|
|
62
|
+
prompt=prompt,
|
|
63
|
+
options=options or ["Yes", "No"],
|
|
64
|
+
metadata=metadata,
|
|
65
|
+
n_blinks=n_blinks,
|
|
66
|
+
threshold=threshold,
|
|
67
|
+
force_mode=force_mode,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
def decide(
|
|
71
|
+
self,
|
|
72
|
+
request: Union[DecisionRequest, Dict[str, Any]],
|
|
73
|
+
n_blinks: Optional[int] = None,
|
|
74
|
+
threshold: Optional[float] = None,
|
|
75
|
+
force_mode: Optional[str] = None,
|
|
76
|
+
) -> DecisionResult[str]:
|
|
77
|
+
payload = request.model_dump(mode="json") if isinstance(request, DecisionRequest) else request
|
|
78
|
+
params = {}
|
|
79
|
+
if n_blinks is not None:
|
|
80
|
+
params["n_blinks"] = n_blinks
|
|
81
|
+
if threshold is not None:
|
|
82
|
+
params["threshold"] = threshold
|
|
83
|
+
if force_mode is not None:
|
|
84
|
+
params["force_mode"] = force_mode
|
|
85
|
+
|
|
86
|
+
resp = self.client.post("/v1/decide", json=payload, params=params)
|
|
87
|
+
resp.raise_for_status()
|
|
88
|
+
return DecisionResult.model_validate(resp.json())
|
|
89
|
+
|
|
90
|
+
def decide_batch(self, requests: List[Union[DecisionRequest, Dict[str, Any]]]) -> List[DecisionResult[str]]:
|
|
91
|
+
payload = {
|
|
92
|
+
"requests": [
|
|
93
|
+
r.model_dump(mode="json") if isinstance(r, DecisionRequest) else r
|
|
94
|
+
for r in requests
|
|
95
|
+
]
|
|
96
|
+
}
|
|
97
|
+
resp = self.client.post("/v1/decide/batch", json=payload)
|
|
98
|
+
resp.raise_for_status()
|
|
99
|
+
return [DecisionResult.model_validate(item) for item in resp.json()["results"]]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class AsyncBlinkThinkRemoteClient:
|
|
103
|
+
def __init__(
|
|
104
|
+
self,
|
|
105
|
+
base_url: str = "http://localhost:8000",
|
|
106
|
+
api_key: Optional[str] = None,
|
|
107
|
+
timeout: float = 60.0,
|
|
108
|
+
):
|
|
109
|
+
self.base_url = base_url.rstrip("/")
|
|
110
|
+
headers = {"Content-Type": "application/json"}
|
|
111
|
+
if api_key:
|
|
112
|
+
headers["Authorization"] = f"Bearer {api_key}"
|
|
113
|
+
self.client = httpx.AsyncClient(base_url=self.base_url, headers=headers, timeout=timeout)
|
|
114
|
+
|
|
115
|
+
async def __aenter__(self):
|
|
116
|
+
return self
|
|
117
|
+
|
|
118
|
+
async def __aexit__(self, *args):
|
|
119
|
+
await self.aclose()
|
|
120
|
+
|
|
121
|
+
async def aclose(self):
|
|
122
|
+
await self.client.aclose()
|
|
123
|
+
|
|
124
|
+
async def choice(
|
|
125
|
+
self,
|
|
126
|
+
prompt: str,
|
|
127
|
+
options: List[str],
|
|
128
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
129
|
+
n_blinks: Optional[int] = None,
|
|
130
|
+
threshold: Optional[float] = None,
|
|
131
|
+
force_mode: Optional[str] = None,
|
|
132
|
+
**kwargs: Any,
|
|
133
|
+
) -> DecisionResult[str]:
|
|
134
|
+
req = DecisionRequest(
|
|
135
|
+
prompt=prompt,
|
|
136
|
+
decision_type=DecisionType.CHOICE,
|
|
137
|
+
options=options,
|
|
138
|
+
metadata=metadata,
|
|
139
|
+
n_blinks=n_blinks,
|
|
140
|
+
threshold=threshold,
|
|
141
|
+
force_mode=force_mode,
|
|
142
|
+
**kwargs,
|
|
143
|
+
)
|
|
144
|
+
return await self.decide(req, n_blinks=n_blinks, threshold=threshold, force_mode=force_mode)
|
|
145
|
+
|
|
146
|
+
async def binary(
|
|
147
|
+
self,
|
|
148
|
+
prompt: str,
|
|
149
|
+
options: Optional[List[str]] = None,
|
|
150
|
+
metadata: Optional[Dict[str, Any]] = None,
|
|
151
|
+
n_blinks: Optional[int] = None,
|
|
152
|
+
threshold: Optional[float] = None,
|
|
153
|
+
force_mode: Optional[str] = None,
|
|
154
|
+
**kwargs: Any,
|
|
155
|
+
) -> DecisionResult[str]:
|
|
156
|
+
return await self.choice(
|
|
157
|
+
prompt=prompt,
|
|
158
|
+
options=options or ["Yes", "No"],
|
|
159
|
+
metadata=metadata,
|
|
160
|
+
n_blinks=n_blinks,
|
|
161
|
+
threshold=threshold,
|
|
162
|
+
force_mode=force_mode,
|
|
163
|
+
**kwargs,
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
async def decide(self, request: Union[DecisionRequest, Dict[str, Any]], **params) -> DecisionResult[str]:
|
|
167
|
+
payload = request.model_dump(mode="json") if isinstance(request, DecisionRequest) else request
|
|
168
|
+
resp = await self.client.post("/v1/decide", json=payload, params={k: v for k, v in params.items() if v is not None})
|
|
169
|
+
resp.raise_for_status()
|
|
170
|
+
return DecisionResult.model_validate(resp.json())
|
|
171
|
+
|
|
172
|
+
async def decide_batch(self, requests: List[Union[DecisionRequest, Dict[str, Any]]]) -> List[DecisionResult[str]]:
|
|
173
|
+
payload = {
|
|
174
|
+
"requests": [
|
|
175
|
+
r.model_dump(mode="json") if isinstance(r, DecisionRequest) else r
|
|
176
|
+
for r in requests
|
|
177
|
+
]
|
|
178
|
+
}
|
|
179
|
+
resp = await self.client.post("/v1/decide/batch", json=payload)
|
|
180
|
+
resp.raise_for_status()
|
|
181
|
+
return [DecisionResult.model_validate(item) for item in resp.json()["results"]]
|
blinkthink/router.py
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from typing import Any, Dict, Literal, Optional, Union
|
|
3
|
+
import torch
|
|
4
|
+
from .calibrator import TemperatureCalibrator
|
|
5
|
+
from .system1 import BlinkEngine
|
|
6
|
+
from .system2 import ThinkEngine
|
|
7
|
+
from .schemas import DecisionRequest, DecisionResult
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class BlinkThinkRouter:
|
|
11
|
+
"""
|
|
12
|
+
Main API interface for BlinkThink dual-process routing.
|
|
13
|
+
"""
|
|
14
|
+
def __init__(
|
|
15
|
+
self,
|
|
16
|
+
model,
|
|
17
|
+
tokenizer,
|
|
18
|
+
n_blinks: int = 2,
|
|
19
|
+
threshold: float = 0.35,
|
|
20
|
+
strict_consensus: bool = False,
|
|
21
|
+
calibrator: Optional[TemperatureCalibrator] = None,
|
|
22
|
+
max_think_tokens: int = 512,
|
|
23
|
+
device: Optional[Union[str, torch.device]] = None,
|
|
24
|
+
prompt_mode: Literal["auto", "chat", "completion"] = "auto",
|
|
25
|
+
confidence_metric: Literal["composite", "margin", "msp"] = "composite",
|
|
26
|
+
):
|
|
27
|
+
if getattr(tokenizer, "pad_token", None) is None:
|
|
28
|
+
if getattr(tokenizer, "eos_token", None) is not None:
|
|
29
|
+
tokenizer.pad_token = tokenizer.eos_token
|
|
30
|
+
|
|
31
|
+
self.calibrator = calibrator or TemperatureCalibrator()
|
|
32
|
+
self.confidence_metric = confidence_metric
|
|
33
|
+
self.blink_engine = BlinkEngine(
|
|
34
|
+
model,
|
|
35
|
+
tokenizer,
|
|
36
|
+
self.calibrator,
|
|
37
|
+
device=device,
|
|
38
|
+
prompt_mode=prompt_mode,
|
|
39
|
+
confidence_metric=confidence_metric,
|
|
40
|
+
)
|
|
41
|
+
self.think_engine = ThinkEngine(
|
|
42
|
+
model, tokenizer, calibrator=self.calibrator, max_new_tokens=max_think_tokens, use_calibrator_temperature=False, device=device
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
self.n_blinks = n_blinks
|
|
46
|
+
self.threshold = threshold
|
|
47
|
+
self.strict_consensus = strict_consensus
|
|
48
|
+
|
|
49
|
+
def decide(
|
|
50
|
+
self,
|
|
51
|
+
request: Union[DecisionRequest, Dict[str, Any]],
|
|
52
|
+
n_blinks: Optional[int] = None,
|
|
53
|
+
threshold: Optional[float] = None,
|
|
54
|
+
force_mode: Optional[Literal["blink_only", "deliberate", "force_deliberate", "system_1", "system_2", "think_only"]] = None,
|
|
55
|
+
) -> DecisionResult[str]:
|
|
56
|
+
if isinstance(request, dict):
|
|
57
|
+
request = DecisionRequest(**request)
|
|
58
|
+
|
|
59
|
+
active_mode = force_mode if force_mode is not None else getattr(request, "force_mode", None)
|
|
60
|
+
active_blinks = n_blinks if n_blinks is not None else (getattr(request, "n_blinks", None) or self.n_blinks)
|
|
61
|
+
active_threshold = threshold if threshold is not None else (
|
|
62
|
+
getattr(request, "threshold", None) if getattr(request, "threshold", None) is not None else self.threshold
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
active_metric = getattr(request, "confidence_metric", None) or self.confidence_metric
|
|
66
|
+
|
|
67
|
+
if active_mode in ("blink_only", "system_1"):
|
|
68
|
+
return self.blink_engine.decide(
|
|
69
|
+
request,
|
|
70
|
+
n_blinks=active_blinks,
|
|
71
|
+
strict_consensus=self.strict_consensus,
|
|
72
|
+
confidence_metric=active_metric,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
blink_res = self.blink_engine.decide(
|
|
76
|
+
request,
|
|
77
|
+
n_blinks=active_blinks,
|
|
78
|
+
strict_consensus=self.strict_consensus,
|
|
79
|
+
confidence_metric=active_metric,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
is_forced_deliberation = active_mode in ("deliberate", "force_deliberate", "system_2", "think_only")
|
|
83
|
+
if not is_forced_deliberation and blink_res.confidence >= active_threshold:
|
|
84
|
+
return blink_res
|
|
85
|
+
|
|
86
|
+
think_res = self.think_engine.decide(request, fallback=blink_res)
|
|
87
|
+
think_res.latency_ms += blink_res.latency_ms
|
|
88
|
+
if think_res.telemetry is None:
|
|
89
|
+
think_res.telemetry = blink_res.telemetry
|
|
90
|
+
|
|
91
|
+
return think_res
|
|
92
|
+
|
|
93
|
+
def save_config(self, filepath: str):
|
|
94
|
+
config = {
|
|
95
|
+
"threshold": self.threshold,
|
|
96
|
+
"n_blinks": self.n_blinks,
|
|
97
|
+
"strict_consensus": self.strict_consensus,
|
|
98
|
+
"temperatures": self.calibrator.to_dict(),
|
|
99
|
+
}
|
|
100
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
|
101
|
+
json.dump(config, f, indent=2)
|
|
102
|
+
|
|
103
|
+
def load_config(self, filepath: str):
|
|
104
|
+
with open(filepath, "r", encoding="utf-8") as f:
|
|
105
|
+
config = json.load(f)
|
|
106
|
+
self.threshold = config.get("threshold", self.threshold)
|
|
107
|
+
self.n_blinks = config.get("n_blinks", self.n_blinks)
|
|
108
|
+
self.strict_consensus = config.get("strict_consensus", self.strict_consensus)
|
|
109
|
+
if "temperatures" in config:
|
|
110
|
+
self.calibrator.load_dict(config["temperatures"])
|