quantum-mind 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- quantum_mind/__init__.py +25 -0
- quantum_mind/_optional.py +14 -0
- quantum_mind/applications/README.md +19 -0
- quantum_mind/applications/__init__.py +1 -0
- quantum_mind/applications/calibration.py +477 -0
- quantum_mind/applications/fusion.py +259 -0
- quantum_mind/applications/handover.py +197 -0
- quantum_mind/applications/intent.py +196 -0
- quantum_mind/applications/orchestration.py +381 -0
- quantum_mind/applications/personalisation.py +242 -0
- quantum_mind/applications/questioning.py +339 -0
- quantum_mind/applications/robotics.py +428 -0
- quantum_mind/circuits/README.md +50 -0
- quantum_mind/circuits/__init__.py +5 -0
- quantum_mind/circuits/backends.py +198 -0
- quantum_mind/circuits/builders.py +414 -0
- quantum_mind/core/__init__.py +4 -0
- quantum_mind/core/fit.py +370 -0
- quantum_mind/core/linalg.py +341 -0
- quantum_mind/core/model.py +283 -0
- quantum_mind/data/__init__.py +58 -0
- quantum_mind/envs/__init__.py +14 -0
- quantum_mind/envs/hri.py +329 -0
- quantum_mind/families/__init__.py +8 -0
- quantum_mind/families/concepts/README.md +20 -0
- quantum_mind/families/concepts/__init__.py +3 -0
- quantum_mind/families/concepts/models.py +148 -0
- quantum_mind/families/conjunction/README.md +58 -0
- quantum_mind/families/conjunction/__init__.py +3 -0
- quantum_mind/families/conjunction/models.py +182 -0
- quantum_mind/families/contextuality/README.md +30 -0
- quantum_mind/families/contextuality/__init__.py +2 -0
- quantum_mind/families/contextuality/models.py +129 -0
- quantum_mind/families/decision/README.md +42 -0
- quantum_mind/families/decision/__init__.py +2 -0
- quantum_mind/families/decision/models.py +171 -0
- quantum_mind/families/dynamics/README.md +51 -0
- quantum_mind/families/dynamics/__init__.py +3 -0
- quantum_mind/families/dynamics/models.py +474 -0
- quantum_mind/families/game_theory/README.md +24 -0
- quantum_mind/families/game_theory/__init__.py +2 -0
- quantum_mind/families/game_theory/models.py +225 -0
- quantum_mind/families/interference/README.md +59 -0
- quantum_mind/families/interference/__init__.py +3 -0
- quantum_mind/families/interference/models.py +129 -0
- quantum_mind/families/memory/README.md +18 -0
- quantum_mind/families/memory/__init__.py +2 -0
- quantum_mind/families/memory/models.py +132 -0
- quantum_mind/families/order_effects/README.md +73 -0
- quantum_mind/families/order_effects/__init__.py +3 -0
- quantum_mind/families/order_effects/models.py +362 -0
- quantum_mind/families/perception/README.md +23 -0
- quantum_mind/families/perception/__init__.py +3 -0
- quantum_mind/families/perception/models.py +220 -0
- quantum_mind/families/qlbn/README.md +56 -0
- quantum_mind/families/qlbn/__init__.py +3 -0
- quantum_mind/families/qlbn/models.py +273 -0
- quantum_mind/families/similarity/README.md +35 -0
- quantum_mind/families/similarity/__init__.py +2 -0
- quantum_mind/families/similarity/models.py +200 -0
- quantum_mind/inspired/README.md +38 -0
- quantum_mind/inspired/__init__.py +27 -0
- quantum_mind/inspired/exploration.py +347 -0
- quantum_mind/inspired/optimisers.py +300 -0
- quantum_mind/inspired/rl.py +306 -0
- quantum_mind/inspired/tensor.py +299 -0
- quantum_mind/inspired/tensor_layers.py +270 -0
- quantum_mind/inspired/text.py +233 -0
- quantum_mind/problems/README.md +20 -0
- quantum_mind/problems/__init__.py +288 -0
- quantum_mind/quantum/README.md +62 -0
- quantum_mind/quantum/__init__.py +41 -0
- quantum_mind/quantum/algorithms.py +493 -0
- quantum_mind/quantum/ansatz.py +208 -0
- quantum_mind/quantum/classifiers.py +692 -0
- quantum_mind/quantum/estimation.py +224 -0
- quantum_mind/quantum/fourier.py +145 -0
- quantum_mind/quantum/policy.py +210 -0
- quantum_mind/quantum/quanvolution.py +173 -0
- quantum_mind/quantum/statevector.py +566 -0
- quantum_mind/quantum/walks.py +170 -0
- quantum_mind/viz/README.md +44 -0
- quantum_mind/viz/__init__.py +32 -0
- quantum_mind/viz/interactive.py +334 -0
- quantum_mind/viz/mpl.py +350 -0
- quantum_mind/viz/states.py +447 -0
- quantum_mind/viz/themes.py +30 -0
- quantum_mind-2.0.0.dist-info/METADATA +600 -0
- quantum_mind-2.0.0.dist-info/RECORD +93 -0
- quantum_mind-2.0.0.dist-info/WHEEL +5 -0
- quantum_mind-2.0.0.dist-info/licenses/LICENSE +201 -0
- quantum_mind-2.0.0.dist-info/licenses/NOTICE +5 -0
- quantum_mind-2.0.0.dist-info/top_level.txt +1 -0
quantum_mind/__init__.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""quantum_mind: quantum-like, quantum and quantum-inspired models, with classical baselines, Qiskit circuits
|
|
2
|
+
and a 3D Bloch-sphere viewer.
|
|
3
|
+
|
|
4
|
+
======================== ===========================================================================
|
|
5
|
+
Subpackage Contents
|
|
6
|
+
======================== ===========================================================================
|
|
7
|
+
:mod:`quantum_mind.core` linear algebra, the :class:`~quantum_mind.core.Model` base class, fitting,
|
|
8
|
+
comparison and model recovery
|
|
9
|
+
:mod:`quantum_mind.families` quantum-like models of judgement and decision, eleven families with
|
|
10
|
+
classical baselines
|
|
11
|
+
:mod:`quantum_mind.applications` robotics: question domains, questioning designs, trust, intent resolution,
|
|
12
|
+
human-model ensemble
|
|
13
|
+
:mod:`quantum_mind.quantum` quantum machine learning and algorithms on a batched simulator
|
|
14
|
+
:mod:`quantum_mind.inspired` quantum-inspired algorithms that run on ordinary hardware
|
|
15
|
+
:mod:`quantum_mind.problems` QUBO builders shared by the quantum and quantum-inspired solvers
|
|
16
|
+
:mod:`quantum_mind.circuits` Qiskit circuits for the families and ``run()`` for simulators and hardware
|
|
17
|
+
:mod:`quantum_mind.viz` Bloch-sphere viewer: trajectories, animations, interactive HTML, live updates
|
|
18
|
+
:mod:`quantum_mind.data` published aggregate data sets
|
|
19
|
+
======================== ===========================================================================
|
|
20
|
+
|
|
21
|
+
The most used functions are re-exported at the top level: :func:`fit`, :func:`compare`,
|
|
22
|
+
:func:`recovery`, :class:`Model` and :class:`Param`.
|
|
23
|
+
"""
|
|
24
|
+
__version__ = '2.0.0'
|
|
25
|
+
from .core import fit, compare, recovery, Model, Param # noqa: F401
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""Clear error messages for optional dependencies."""
|
|
2
|
+
import importlib
|
|
3
|
+
|
|
4
|
+
_EXTRA = {'qiskit': 'qiskit', 'qiskit_aer': 'qiskit', 'qiskit_ibm_runtime': 'qiskit', 'braket': 'cloud',
|
|
5
|
+
'matplotlib': 'viz', 'plotly': 'viz', 'ipywidgets': 'viz', 'anywidget': 'viz', 'seaborn': 'viz'}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def require(module, extra=None):
|
|
9
|
+
"""Import `module` or raise ImportError naming the pip extra that provides it."""
|
|
10
|
+
try:
|
|
11
|
+
return importlib.import_module(module)
|
|
12
|
+
except ImportError as e:
|
|
13
|
+
extra = extra or _EXTRA.get(module.split('.')[0], 'all')
|
|
14
|
+
raise ImportError('%s is needed for this feature: pip install "quantum-mind[%s]"' % (module, extra)) from e
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# Applications
|
|
2
|
+
|
|
3
|
+
## `quantum_mind.applications.robotics`: robots that ask people questions
|
|
4
|
+
|
|
5
|
+
| Name | Purpose |
|
|
6
|
+
|---|---|
|
|
7
|
+
| `HRI_DOMAINS`, `domain_models(domain)` | Three question pairs (object clarification, trust and hand-over, preference elicitation) with quantum-like, anchoring and Bayesian populations (illustrative parameters) |
|
|
8
|
+
| `estimate_unprimed_rates(design, generator, n, model, probe)` | Questioning designs: fixed order, reversed-order probe with a model, split order (model-free) |
|
|
9
|
+
| `TRUST_PROTOCOL`, `TRUST_MODELS` | Six hand-overs, with the trust question at the end only or also midway |
|
|
10
|
+
| `HumanModelEnsemble` | Competing human models weighted by BIC; predicted answer distribution plus uncertainty (entropy and model disagreement, in bits) for use as a confidence signal |
|
|
11
|
+
| `ask_or_act(p_success, uncertainty, ask_cost, error_cost, max_disagreement_bits)` | Decision rule: ask when the human models disagree or when the expected cost of a wrong action exceeds the cost of asking |
|
|
12
|
+
| `QuantumIntentResolver`, `BayesIntentResolver` (`quantum_mind.applications.intent`) | Intent from ambiguous commands and context cues; cues as Kraus operators rotated by an incompatibility angle, so cue order matters; every angle 0 gives exactly Bayes' rule |
|
|
13
|
+
| `HumanModelService` | Answers in, prediction + uncertainty + ask/act decision out (JSON-friendly dicts); wrapped by the ROS 2 node in [`integrations/ros2`](../../../integrations/ros2/README.md) |
|
|
14
|
+
|
|
15
|
+
Findings from the companion simulation study ("Order-Aware Human Models for Robot Questioning and Trust"):
|
|
16
|
+
splitting the question order beat model-based correction for recovering unprimed answers; the
|
|
17
|
+
quantum-like model was the best predictor only when people followed it and were alike; asking about
|
|
18
|
+
trust midway changed final trust under open-system dynamics (−0.076) and needed about 1,600 people per
|
|
19
|
+
condition to detect. Use the ensemble rather than a single model.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Ready-made applications of the model families. See README.md in this folder."""
|
|
@@ -0,0 +1,477 @@
|
|
|
1
|
+
r"""Calibration of confidence signals: measure it, correct it, and bound it.
|
|
2
|
+
|
|
3
|
+
A robot that routes between modules (vision, language, a human model, a planner) or decides whether to
|
|
4
|
+
ask a person before acting relies on the confidence each module reports. A confidence is *calibrated*
|
|
5
|
+
when, among all the times a module reports 0.8, it is right about 80% of the time. This module provides
|
|
6
|
+
the standard tools to check and repair that property.
|
|
7
|
+
|
|
8
|
+
* Measures: :func:`reliability_curve`, :func:`expected_calibration_error`,
|
|
9
|
+
:func:`max_calibration_error`, :func:`brier_score`.
|
|
10
|
+
* Recalibration: :class:`PlattScaling` (binary), :class:`TemperatureScaling` (multi-class),
|
|
11
|
+
:class:`IsotonicCalibration` (monotone, non-parametric).
|
|
12
|
+
* Guarantees: :class:`SplitConformalClassifier` (prediction sets with a coverage guarantee) and
|
|
13
|
+
:func:`clopper_pearson` (exact binomial bounds), combined in :class:`CalibrationMonitor`, which tracks
|
|
14
|
+
a module's outcomes online and returns a lower confidence bound on its success probability.
|
|
15
|
+
|
|
16
|
+
Examples
|
|
17
|
+
--------
|
|
18
|
+
>>> import numpy as np
|
|
19
|
+
>>> from quantum_mind.applications.calibration import expected_calibration_error, IsotonicCalibration
|
|
20
|
+
>>> rng = np.random.default_rng(0)
|
|
21
|
+
>>> p = rng.uniform(0.3, 0.9, 5000) # true success probabilities
|
|
22
|
+
>>> y = rng.random(5000) < p # outcomes
|
|
23
|
+
>>> over = np.clip(p + 0.15, 0, 1) # an over-confident module
|
|
24
|
+
>>> expected_calibration_error(over, y) > 0.1 # reports about 0.15 too much
|
|
25
|
+
True
|
|
26
|
+
>>> fixed = IsotonicCalibration().fit(over, y).transform(over)
|
|
27
|
+
>>> expected_calibration_error(fixed, y) < 0.03
|
|
28
|
+
True
|
|
29
|
+
"""
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
from dataclasses import dataclass, field
|
|
33
|
+
import numpy as np
|
|
34
|
+
from scipy.optimize import minimize, minimize_scalar
|
|
35
|
+
from scipy.special import expit, logit
|
|
36
|
+
from scipy.stats import beta
|
|
37
|
+
|
|
38
|
+
__all__ = ['reliability_curve', 'expected_calibration_error', 'max_calibration_error', 'brier_score',
|
|
39
|
+
'PlattScaling', 'TemperatureScaling', 'IsotonicCalibration', 'SplitConformalClassifier',
|
|
40
|
+
'clopper_pearson', 'CalibrationMonitor']
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _binary_inputs(conf, outcome):
|
|
44
|
+
"""Validate a vector of confidences in [0, 1] and matching 0/1 outcomes."""
|
|
45
|
+
c = np.asarray(conf, float).ravel()
|
|
46
|
+
y = np.asarray(outcome, float).ravel()
|
|
47
|
+
if c.shape != y.shape or c.size == 0:
|
|
48
|
+
raise ValueError('confidences and outcomes must be non-empty and of the same length')
|
|
49
|
+
if np.any((c < 0) | (c > 1)):
|
|
50
|
+
raise ValueError('confidences must lie in [0, 1]')
|
|
51
|
+
return c, y
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def reliability_curve(conf, outcome, bins=10):
|
|
55
|
+
"""Data of a reliability diagram.
|
|
56
|
+
|
|
57
|
+
Parameters
|
|
58
|
+
----------
|
|
59
|
+
conf : array_like
|
|
60
|
+
Reported confidences in [0, 1].
|
|
61
|
+
outcome : array_like
|
|
62
|
+
1 if the prediction was right (or the action succeeded), 0 otherwise.
|
|
63
|
+
bins : int, optional
|
|
64
|
+
Number of equal-width confidence bins.
|
|
65
|
+
|
|
66
|
+
Returns
|
|
67
|
+
-------
|
|
68
|
+
dict
|
|
69
|
+
``confidence`` (mean confidence per bin), ``accuracy`` (success rate per bin), ``count`` and
|
|
70
|
+
``edges``. Empty bins hold ``nan``. A calibrated module lies on the diagonal.
|
|
71
|
+
"""
|
|
72
|
+
c, y = _binary_inputs(conf, outcome)
|
|
73
|
+
edges = np.linspace(0, 1, bins + 1)
|
|
74
|
+
idx = np.clip(np.digitize(c, edges[1:-1]), 0, bins - 1)
|
|
75
|
+
cnt = np.bincount(idx, minlength=bins).astype(float)
|
|
76
|
+
with np.errstate(invalid='ignore', divide='ignore'):
|
|
77
|
+
mc = np.bincount(idx, weights=c, minlength=bins) / cnt
|
|
78
|
+
acc = np.bincount(idx, weights=y, minlength=bins) / cnt
|
|
79
|
+
return {'confidence': mc, 'accuracy': acc, 'count': cnt.astype(int), 'edges': edges}
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def expected_calibration_error(conf, outcome, bins=10):
|
|
83
|
+
"""Expected calibration error (ECE).
|
|
84
|
+
|
|
85
|
+
Parameters
|
|
86
|
+
----------
|
|
87
|
+
conf : array_like
|
|
88
|
+
Confidences in [0, 1].
|
|
89
|
+
outcome : array_like
|
|
90
|
+
0/1 outcomes.
|
|
91
|
+
bins : int, optional
|
|
92
|
+
Number of bins.
|
|
93
|
+
|
|
94
|
+
Returns
|
|
95
|
+
-------
|
|
96
|
+
float
|
|
97
|
+
:math:`\\sum_b \\frac{n_b}{N} |\\mathrm{acc}_b - \\mathrm{conf}_b|`; 0 for a perfectly calibrated module.
|
|
98
|
+
"""
|
|
99
|
+
r = reliability_curve(conf, outcome, bins)
|
|
100
|
+
ok = r['count'] > 0
|
|
101
|
+
return float(np.sum(r['count'][ok] * np.abs(r['accuracy'][ok] - r['confidence'][ok])) / r['count'].sum())
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def max_calibration_error(conf, outcome, bins=10):
|
|
105
|
+
"""Largest calibration gap over the non-empty bins.
|
|
106
|
+
|
|
107
|
+
Parameters
|
|
108
|
+
----------
|
|
109
|
+
conf : array_like
|
|
110
|
+
Confidences.
|
|
111
|
+
outcome : array_like
|
|
112
|
+
0/1 outcomes.
|
|
113
|
+
bins : int, optional
|
|
114
|
+
Number of bins.
|
|
115
|
+
|
|
116
|
+
Returns
|
|
117
|
+
-------
|
|
118
|
+
float
|
|
119
|
+
"""
|
|
120
|
+
r = reliability_curve(conf, outcome, bins)
|
|
121
|
+
ok = r['count'] > 0
|
|
122
|
+
return float(np.max(np.abs(r['accuracy'][ok] - r['confidence'][ok])))
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def brier_score(prob, outcome):
|
|
126
|
+
"""Brier score (mean squared error of probabilities).
|
|
127
|
+
|
|
128
|
+
Parameters
|
|
129
|
+
----------
|
|
130
|
+
prob : array_like
|
|
131
|
+
Binary confidences, shape ``(n,)``, or class probabilities, shape ``(n, k)``.
|
|
132
|
+
outcome : array_like
|
|
133
|
+
0/1 outcomes, shape ``(n,)``, or class labels ``0..k-1`` for the multi-class case.
|
|
134
|
+
|
|
135
|
+
Returns
|
|
136
|
+
-------
|
|
137
|
+
float
|
|
138
|
+
Lower is better; it rewards both calibration and sharpness.
|
|
139
|
+
"""
|
|
140
|
+
p = np.asarray(prob, float)
|
|
141
|
+
if p.ndim == 1:
|
|
142
|
+
c, y = _binary_inputs(p, outcome)
|
|
143
|
+
return float(np.mean((c - y) ** 2))
|
|
144
|
+
Y = np.eye(p.shape[1])[np.asarray(outcome, int)]
|
|
145
|
+
return float(np.mean(np.sum((p - Y) ** 2, axis=1)))
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@dataclass
|
|
149
|
+
class PlattScaling:
|
|
150
|
+
"""Logistic recalibration of a binary confidence: :math:`\\sigma(a\\,\\mathrm{logit}(c) + b)`.
|
|
151
|
+
|
|
152
|
+
Attributes
|
|
153
|
+
----------
|
|
154
|
+
a, b : float
|
|
155
|
+
Slope and intercept, fitted by maximum likelihood (``a = 1, b = 0`` is the identity).
|
|
156
|
+
"""
|
|
157
|
+
a: float = 1.0
|
|
158
|
+
b: float = 0.0
|
|
159
|
+
|
|
160
|
+
def fit(self, conf, outcome):
|
|
161
|
+
"""Fit the slope and intercept.
|
|
162
|
+
|
|
163
|
+
Parameters
|
|
164
|
+
----------
|
|
165
|
+
conf : array_like
|
|
166
|
+
Confidences in [0, 1].
|
|
167
|
+
outcome : array_like
|
|
168
|
+
0/1 outcomes.
|
|
169
|
+
|
|
170
|
+
Returns
|
|
171
|
+
-------
|
|
172
|
+
PlattScaling
|
|
173
|
+
``self``.
|
|
174
|
+
"""
|
|
175
|
+
c, y = _binary_inputs(conf, outcome)
|
|
176
|
+
z = logit(np.clip(c, 1e-6, 1 - 1e-6))
|
|
177
|
+
|
|
178
|
+
def nll(w):
|
|
179
|
+
s = w[0] * z + w[1]
|
|
180
|
+
return float(np.sum(np.logaddexp(0, s) - y * s))
|
|
181
|
+
self.a, self.b = (float(v) for v in minimize(nll, [1.0, 0.0], method='BFGS').x)
|
|
182
|
+
return self
|
|
183
|
+
|
|
184
|
+
def transform(self, conf):
|
|
185
|
+
"""Recalibrated confidences.
|
|
186
|
+
|
|
187
|
+
Parameters
|
|
188
|
+
----------
|
|
189
|
+
conf : array_like
|
|
190
|
+
Confidences in [0, 1].
|
|
191
|
+
|
|
192
|
+
Returns
|
|
193
|
+
-------
|
|
194
|
+
numpy.ndarray
|
|
195
|
+
"""
|
|
196
|
+
z = logit(np.clip(np.asarray(conf, float), 1e-6, 1 - 1e-6))
|
|
197
|
+
return expit(self.a * z + self.b)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
@dataclass
|
|
201
|
+
class TemperatureScaling:
|
|
202
|
+
"""Multi-class recalibration by one temperature: :math:`\\mathrm{softmax}(\\log p / T)`.
|
|
203
|
+
|
|
204
|
+
Attributes
|
|
205
|
+
----------
|
|
206
|
+
T : float
|
|
207
|
+
Temperature (> 1 softens over-confident predictions, < 1 sharpens under-confident ones).
|
|
208
|
+
"""
|
|
209
|
+
T: float = 1.0
|
|
210
|
+
|
|
211
|
+
@staticmethod
|
|
212
|
+
def _scaled(prob, T):
|
|
213
|
+
"""Softmax of log-probabilities divided by T."""
|
|
214
|
+
z = np.log(np.clip(np.asarray(prob, float), 1e-12, None)) / T
|
|
215
|
+
z -= z.max(axis=1, keepdims=True)
|
|
216
|
+
e = np.exp(z)
|
|
217
|
+
return e / e.sum(axis=1, keepdims=True)
|
|
218
|
+
|
|
219
|
+
def fit(self, prob, labels):
|
|
220
|
+
"""Fit the temperature by minimising the negative log-likelihood.
|
|
221
|
+
|
|
222
|
+
Parameters
|
|
223
|
+
----------
|
|
224
|
+
prob : array_like
|
|
225
|
+
Class probabilities, shape ``(n, k)``.
|
|
226
|
+
labels : array_like
|
|
227
|
+
True classes ``0..k-1``.
|
|
228
|
+
|
|
229
|
+
Returns
|
|
230
|
+
-------
|
|
231
|
+
TemperatureScaling
|
|
232
|
+
``self``.
|
|
233
|
+
"""
|
|
234
|
+
p = np.asarray(prob, float)
|
|
235
|
+
y = np.asarray(labels, int)
|
|
236
|
+
|
|
237
|
+
def nll(logT):
|
|
238
|
+
q = self._scaled(p, np.exp(logT))
|
|
239
|
+
return -float(np.sum(np.log(np.clip(q[np.arange(len(y)), y], 1e-12, None))))
|
|
240
|
+
self.T = float(np.exp(minimize_scalar(nll, bounds=(-3, 3), method='bounded').x))
|
|
241
|
+
return self
|
|
242
|
+
|
|
243
|
+
def transform(self, prob):
|
|
244
|
+
"""Recalibrated class probabilities.
|
|
245
|
+
|
|
246
|
+
Parameters
|
|
247
|
+
----------
|
|
248
|
+
prob : array_like
|
|
249
|
+
Shape ``(n, k)``.
|
|
250
|
+
|
|
251
|
+
Returns
|
|
252
|
+
-------
|
|
253
|
+
numpy.ndarray
|
|
254
|
+
"""
|
|
255
|
+
return self._scaled(prob, self.T)
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@dataclass
|
|
259
|
+
class IsotonicCalibration:
|
|
260
|
+
"""Monotone, non-parametric recalibration by the pool-adjacent-violators algorithm.
|
|
261
|
+
|
|
262
|
+
Attributes
|
|
263
|
+
----------
|
|
264
|
+
x_, y_ : numpy.ndarray
|
|
265
|
+
Knots of the fitted non-decreasing step function (interpolated linearly between knots).
|
|
266
|
+
"""
|
|
267
|
+
x_: np.ndarray = field(default=None, repr=False)
|
|
268
|
+
y_: np.ndarray = field(default=None, repr=False)
|
|
269
|
+
|
|
270
|
+
def fit(self, conf, outcome):
|
|
271
|
+
"""Fit the monotone map from confidence to success rate.
|
|
272
|
+
|
|
273
|
+
Parameters
|
|
274
|
+
----------
|
|
275
|
+
conf : array_like
|
|
276
|
+
Confidences in [0, 1].
|
|
277
|
+
outcome : array_like
|
|
278
|
+
0/1 outcomes.
|
|
279
|
+
|
|
280
|
+
Returns
|
|
281
|
+
-------
|
|
282
|
+
IsotonicCalibration
|
|
283
|
+
``self``.
|
|
284
|
+
"""
|
|
285
|
+
c, y = _binary_inputs(conf, outcome)
|
|
286
|
+
order = np.argsort(c)
|
|
287
|
+
c, y = c[order], y[order]
|
|
288
|
+
# pool adjacent violators: blocks of (sum, count, x-mean)
|
|
289
|
+
sums, cnts, xs = [], [], []
|
|
290
|
+
for ci, yi in zip(c, y):
|
|
291
|
+
sums.append(yi); cnts.append(1.0); xs.append(ci)
|
|
292
|
+
while len(sums) > 1 and sums[-2] / cnts[-2] > sums[-1] / cnts[-1]:
|
|
293
|
+
s, n, x = sums.pop(), cnts.pop(), xs.pop()
|
|
294
|
+
xs[-1] = (xs[-1] * cnts[-1] + x * n) / (cnts[-1] + n)
|
|
295
|
+
sums[-1] += s
|
|
296
|
+
cnts[-1] += n
|
|
297
|
+
self.x_ = np.array(xs)
|
|
298
|
+
self.y_ = np.array(sums) / np.array(cnts)
|
|
299
|
+
return self
|
|
300
|
+
|
|
301
|
+
def transform(self, conf):
|
|
302
|
+
"""Recalibrated confidences.
|
|
303
|
+
|
|
304
|
+
Parameters
|
|
305
|
+
----------
|
|
306
|
+
conf : array_like
|
|
307
|
+
|
|
308
|
+
Returns
|
|
309
|
+
-------
|
|
310
|
+
numpy.ndarray
|
|
311
|
+
"""
|
|
312
|
+
return np.interp(np.asarray(conf, float), self.x_, self.y_)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@dataclass
|
|
316
|
+
class SplitConformalClassifier:
|
|
317
|
+
"""Split conformal prediction sets for any probabilistic classifier.
|
|
318
|
+
|
|
319
|
+
After :meth:`fit` on held-out calibration data, the prediction set of a new input contains the
|
|
320
|
+
true class with probability at least :math:`1 - \\alpha` (marginally, for exchangeable data),
|
|
321
|
+
whatever the classifier. A robot can act when the set has one element and ask when it has several.
|
|
322
|
+
|
|
323
|
+
Attributes
|
|
324
|
+
----------
|
|
325
|
+
alpha : float
|
|
326
|
+
Allowed miscoverage, for example 0.1 for 90% coverage.
|
|
327
|
+
qhat : float
|
|
328
|
+
Fitted threshold on the score :math:`1 - p_{\\text{true}}`.
|
|
329
|
+
"""
|
|
330
|
+
alpha: float = 0.1
|
|
331
|
+
qhat: float = None
|
|
332
|
+
|
|
333
|
+
def fit(self, prob, labels):
|
|
334
|
+
"""Compute the conformal threshold from calibration data.
|
|
335
|
+
|
|
336
|
+
Parameters
|
|
337
|
+
----------
|
|
338
|
+
prob : array_like
|
|
339
|
+
Class probabilities on the calibration set, shape ``(n, k)``.
|
|
340
|
+
labels : array_like
|
|
341
|
+
True classes.
|
|
342
|
+
|
|
343
|
+
Returns
|
|
344
|
+
-------
|
|
345
|
+
SplitConformalClassifier
|
|
346
|
+
``self``.
|
|
347
|
+
"""
|
|
348
|
+
p = np.asarray(prob, float)
|
|
349
|
+
y = np.asarray(labels, int)
|
|
350
|
+
n = len(y)
|
|
351
|
+
scores = 1 - p[np.arange(n), y]
|
|
352
|
+
level = min(1.0, np.ceil((n + 1) * (1 - self.alpha)) / n)
|
|
353
|
+
self.qhat = float(np.quantile(scores, level, method='higher'))
|
|
354
|
+
return self
|
|
355
|
+
|
|
356
|
+
def predict_sets(self, prob):
|
|
357
|
+
"""Prediction sets.
|
|
358
|
+
|
|
359
|
+
Parameters
|
|
360
|
+
----------
|
|
361
|
+
prob : array_like
|
|
362
|
+
Class probabilities, shape ``(n, k)``.
|
|
363
|
+
|
|
364
|
+
Returns
|
|
365
|
+
-------
|
|
366
|
+
numpy.ndarray
|
|
367
|
+
Boolean mask ``(n, k)``: class j is in the set of sample i when ``mask[i, j]``.
|
|
368
|
+
"""
|
|
369
|
+
return 1 - np.asarray(prob, float) <= self.qhat
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def clopper_pearson(successes, trials, alpha=0.05):
|
|
373
|
+
"""Exact (Clopper-Pearson) confidence interval for a success probability.
|
|
374
|
+
|
|
375
|
+
Parameters
|
|
376
|
+
----------
|
|
377
|
+
successes : int
|
|
378
|
+
Number of successes.
|
|
379
|
+
trials : int
|
|
380
|
+
Number of trials.
|
|
381
|
+
alpha : float, optional
|
|
382
|
+
Two-sided error level.
|
|
383
|
+
|
|
384
|
+
Returns
|
|
385
|
+
-------
|
|
386
|
+
tuple of float
|
|
387
|
+
``(lower, upper)``; ``(0, 1)`` when there are no trials.
|
|
388
|
+
"""
|
|
389
|
+
if trials <= 0:
|
|
390
|
+
return 0.0, 1.0
|
|
391
|
+
lo = 0.0 if successes == 0 else float(beta.ppf(alpha / 2, successes, trials - successes + 1))
|
|
392
|
+
hi = 1.0 if successes == trials else float(beta.ppf(1 - alpha / 2, successes + 1, trials - successes))
|
|
393
|
+
return lo, hi
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
class CalibrationMonitor:
|
|
397
|
+
"""Online calibration check of one module, with a guaranteed lower bound on its success rate.
|
|
398
|
+
|
|
399
|
+
Outcomes are binned by reported confidence. For a new confidence the monitor returns the
|
|
400
|
+
recalibrated success estimate of its bin and a Clopper-Pearson lower bound, which a risk-aware
|
|
401
|
+
decision rule (:func:`quantum_mind.applications.robotics.risk_aware_ask_or_act`) can use instead of the
|
|
402
|
+
raw confidence.
|
|
403
|
+
|
|
404
|
+
Parameters
|
|
405
|
+
----------
|
|
406
|
+
bins : int, optional
|
|
407
|
+
Number of confidence bins.
|
|
408
|
+
alpha : float, optional
|
|
409
|
+
Error level of the bounds.
|
|
410
|
+
min_count : int, optional
|
|
411
|
+
Below this many outcomes in a bin, the raw confidence is returned as the estimate.
|
|
412
|
+
|
|
413
|
+
Examples
|
|
414
|
+
--------
|
|
415
|
+
>>> from quantum_mind.applications.calibration import CalibrationMonitor
|
|
416
|
+
>>> m = CalibrationMonitor()
|
|
417
|
+
>>> for c, ok in [(0.9, 1), (0.9, 0), (0.9, 1), (0.9, 0)] * 10:
|
|
418
|
+
... m.update(c, ok)
|
|
419
|
+
>>> est, lower = m.estimate(0.9)
|
|
420
|
+
>>> round(est, 2), lower < 0.5
|
|
421
|
+
(0.5, True)
|
|
422
|
+
"""
|
|
423
|
+
|
|
424
|
+
def __init__(self, bins=10, alpha=0.05, min_count=10):
|
|
425
|
+
self.bins, self.alpha, self.min_count = bins, alpha, min_count
|
|
426
|
+
self.n = np.zeros(bins, int)
|
|
427
|
+
self.k = np.zeros(bins, int)
|
|
428
|
+
|
|
429
|
+
def _bin(self, conf):
|
|
430
|
+
"""Bin index of a confidence."""
|
|
431
|
+
return int(min(self.bins - 1, max(0, np.floor(float(conf) * self.bins))))
|
|
432
|
+
|
|
433
|
+
def update(self, conf, success):
|
|
434
|
+
"""Record one outcome.
|
|
435
|
+
|
|
436
|
+
Parameters
|
|
437
|
+
----------
|
|
438
|
+
conf : float
|
|
439
|
+
Confidence the module reported.
|
|
440
|
+
success : bool or int
|
|
441
|
+
Whether the module turned out to be right.
|
|
442
|
+
"""
|
|
443
|
+
b = self._bin(conf)
|
|
444
|
+
self.n[b] += 1
|
|
445
|
+
self.k[b] += int(bool(success))
|
|
446
|
+
|
|
447
|
+
def estimate(self, conf):
|
|
448
|
+
"""Recalibrated success estimate and lower bound for a reported confidence.
|
|
449
|
+
|
|
450
|
+
Parameters
|
|
451
|
+
----------
|
|
452
|
+
conf : float
|
|
453
|
+
|
|
454
|
+
Returns
|
|
455
|
+
-------
|
|
456
|
+
estimate : float
|
|
457
|
+
Observed success rate of the bin (the raw confidence while the bin has too few outcomes).
|
|
458
|
+
lower : float
|
|
459
|
+
Clopper-Pearson lower bound (0 while the bin is empty).
|
|
460
|
+
"""
|
|
461
|
+
b = self._bin(conf)
|
|
462
|
+
lo, _ = clopper_pearson(self.k[b], self.n[b], self.alpha)
|
|
463
|
+
est = self.k[b] / self.n[b] if self.n[b] >= self.min_count else float(conf)
|
|
464
|
+
return float(est), float(lo)
|
|
465
|
+
|
|
466
|
+
def ece(self):
|
|
467
|
+
"""Expected calibration error of the outcomes recorded so far (bin centres as confidences).
|
|
468
|
+
|
|
469
|
+
Returns
|
|
470
|
+
-------
|
|
471
|
+
float
|
|
472
|
+
"""
|
|
473
|
+
ok = self.n > 0
|
|
474
|
+
if not ok.any():
|
|
475
|
+
return 0.0
|
|
476
|
+
centres = (np.arange(self.bins) + 0.5) / self.bins
|
|
477
|
+
return float(np.sum(self.n[ok] * np.abs(self.k[ok] / self.n[ok] - centres[ok])) / self.n.sum())
|