CompStats 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CompStats/__init__.py +2 -1
- CompStats/interface.py +481 -0
- CompStats/metrics.py +553 -0
- CompStats/performance.py +29 -20
- CompStats/tests/test_interface.py +153 -0
- CompStats/tests/test_metrics.py +453 -0
- CompStats/tests/test_performance.py +2 -2
- CompStats/utils.py +34 -4
- CompStats-0.1.4.dist-info/METADATA +117 -0
- CompStats-0.1.4.dist-info/RECORD +19 -0
- {CompStats-0.1.2.dist-info → CompStats-0.1.4.dist-info}/WHEEL +1 -1
- CompStats-0.1.2.dist-info/METADATA +0 -52
- CompStats-0.1.2.dist-info/RECORD +0 -15
- {CompStats-0.1.2.dist-info → CompStats-0.1.4.dist-info}/LICENSE +0 -0
- {CompStats-0.1.2.dist-info → CompStats-0.1.4.dist-info}/top_level.txt +0 -0
CompStats/__init__.py
CHANGED
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.4'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
18
18
|
from CompStats.performance import performance_multiple_metrics, difference_multiple, plot_performance_multiple, plot_difference_multiple
|
|
19
19
|
from CompStats.performance import all_differences_multiple, plot_performance2, plot_difference2, plot_scatter_matrix
|
|
20
|
+
from CompStats.interface import Perf
|
CompStats/interface.py
ADDED
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
# Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from sklearn.metrics import balanced_accuracy_score
|
|
16
|
+
from sklearn.base import clone
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import numpy as np
|
|
19
|
+
from CompStats.bootstrap import StatisticSamples
|
|
20
|
+
from CompStats.utils import progress_bar
|
|
21
|
+
from CompStats.measurements import SE
|
|
22
|
+
from CompStats.performance import plot_performance, plot_difference
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Perf(object):
|
|
26
|
+
"""Perf is an entry point to CompStats
|
|
27
|
+
|
|
28
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
29
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
30
|
+
:param score_func: Function to measure the performance, it is assumed that the best algorithm has the highest value.
|
|
31
|
+
:type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
32
|
+
:param error_func: Function to measure the performance where the best algorithm has the lowest value.
|
|
33
|
+
:type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
34
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
35
|
+
:type y_pred: numpy.ndarray
|
|
36
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
37
|
+
:type kwargs: numpy.ndarray
|
|
38
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
39
|
+
:type n_jobs: int
|
|
40
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
41
|
+
:type num_samples: int
|
|
42
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True.
|
|
43
|
+
:type use_tqdm: bool
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
>>> from sklearn.svm import LinearSVC
|
|
47
|
+
>>> from sklearn.linear_model import LogisticRegression
|
|
48
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
49
|
+
>>> from sklearn.datasets import load_iris
|
|
50
|
+
>>> from sklearn.model_selection import train_test_split
|
|
51
|
+
>>> from sklearn.base import clone
|
|
52
|
+
>>> from CompStats.interface import Perf
|
|
53
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
54
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
55
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
56
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
57
|
+
>>> hy = m.predict(X_val)
|
|
58
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
59
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
60
|
+
>>> perf
|
|
61
|
+
<Perf>
|
|
62
|
+
Statistic with its standard error (se)
|
|
63
|
+
statistic (se)
|
|
64
|
+
0.9792 (0.0221) <= alg-1
|
|
65
|
+
0.9744 (0.0246) <= forest
|
|
66
|
+
|
|
67
|
+
If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
|
|
68
|
+
|
|
69
|
+
>>> lr = LogisticRegression().fit(X_train, y_train)
|
|
70
|
+
>>> perf(lr.predict(X_val), name='Log. Reg.')
|
|
71
|
+
<Perf>
|
|
72
|
+
Statistic with its standard error (se)
|
|
73
|
+
statistic (se)
|
|
74
|
+
1.0000 (0.0000) <= Log. Reg.
|
|
75
|
+
0.9792 (0.0221) <= alg-1
|
|
76
|
+
0.9744 (0.0246) <= forest
|
|
77
|
+
|
|
78
|
+
The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
|
|
79
|
+
|
|
80
|
+
>>> perf_error = clone(perf)
|
|
81
|
+
>>> perf_error.error_func = lambda y, hy: (y != hy).mean()
|
|
82
|
+
>>> perf_error
|
|
83
|
+
<Perf>
|
|
84
|
+
Statistic with its standard error (se)
|
|
85
|
+
statistic (se)
|
|
86
|
+
0.0000 (0.0000) <= Log. Reg.
|
|
87
|
+
0.0222 (0.0237) <= alg-1
|
|
88
|
+
0.0222 (0.0215) <= forest
|
|
89
|
+
|
|
90
|
+
"""
|
|
91
|
+
def __init__(self, y_true, *y_pred,
|
|
92
|
+
score_func=balanced_accuracy_score,
|
|
93
|
+
error_func=None,
|
|
94
|
+
num_samples: int=500,
|
|
95
|
+
n_jobs: int=-1,
|
|
96
|
+
use_tqdm=True,
|
|
97
|
+
**kwargs):
|
|
98
|
+
assert (score_func is None) ^ (error_func is None)
|
|
99
|
+
self.score_func = score_func
|
|
100
|
+
self.error_func = error_func
|
|
101
|
+
algs = {}
|
|
102
|
+
for k, v in enumerate(y_pred):
|
|
103
|
+
algs[f'alg-{k+1}'] = np.asanyarray(v)
|
|
104
|
+
algs.update(**kwargs)
|
|
105
|
+
self.predictions = algs
|
|
106
|
+
self.y_true = y_true
|
|
107
|
+
self.num_samples = num_samples
|
|
108
|
+
self.n_jobs = n_jobs
|
|
109
|
+
self.use_tqdm = use_tqdm
|
|
110
|
+
self._init()
|
|
111
|
+
|
|
112
|
+
def _init(self):
|
|
113
|
+
"""Compute the bootstrap statistic"""
|
|
114
|
+
|
|
115
|
+
bib = True if self.score_func is not None else False
|
|
116
|
+
if hasattr(self, '_statistic_samples'):
|
|
117
|
+
_ = self.statistic_samples
|
|
118
|
+
_.BiB = bib
|
|
119
|
+
else:
|
|
120
|
+
_ = StatisticSamples(statistic=self.statistic_func,
|
|
121
|
+
n_jobs=self.n_jobs,
|
|
122
|
+
num_samples=self.num_samples,
|
|
123
|
+
BiB=bib)
|
|
124
|
+
_.samples(N=self.y_true.shape[0])
|
|
125
|
+
self.statistic_samples = _
|
|
126
|
+
|
|
127
|
+
def get_params(self):
|
|
128
|
+
"""Parameters"""
|
|
129
|
+
|
|
130
|
+
return dict(y_true=self.y_true,
|
|
131
|
+
score_func=self.score_func,
|
|
132
|
+
error_func=self.error_func,
|
|
133
|
+
num_samples=self.num_samples,
|
|
134
|
+
n_jobs=self.n_jobs)
|
|
135
|
+
|
|
136
|
+
def __sklearn_clone__(self):
|
|
137
|
+
klass = self.__class__
|
|
138
|
+
params = self.get_params()
|
|
139
|
+
ins = klass(**params)
|
|
140
|
+
ins.predictions = dict(self.predictions)
|
|
141
|
+
ins._statistic_samples._samples = self.statistic_samples._samples
|
|
142
|
+
return ins
|
|
143
|
+
|
|
144
|
+
def __repr__(self):
|
|
145
|
+
"""Prediction statistics with standard error in parenthesis"""
|
|
146
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
147
|
+
|
|
148
|
+
def __str__(self):
|
|
149
|
+
"""Prediction statistics with standard error in parenthesis"""
|
|
150
|
+
|
|
151
|
+
se = self.se
|
|
152
|
+
output = ["Statistic with its standard error (se)"]
|
|
153
|
+
output.append("statistic (se)")
|
|
154
|
+
for key, value in self.statistic.items():
|
|
155
|
+
output.append(f'{value:0.4f} ({se[key]:0.4f}) <= {key}')
|
|
156
|
+
return "\n".join(output)
|
|
157
|
+
|
|
158
|
+
def __call__(self, y_pred, name=None):
|
|
159
|
+
"""Add predictions"""
|
|
160
|
+
if name is None:
|
|
161
|
+
k = len(self.predictions) + 1
|
|
162
|
+
if k == 0:
|
|
163
|
+
k = 1
|
|
164
|
+
name = f'alg-{k}'
|
|
165
|
+
self.predictions[name] = np.asanyarray(y_pred)
|
|
166
|
+
samples = self._statistic_samples
|
|
167
|
+
calls = samples.calls
|
|
168
|
+
if name in calls:
|
|
169
|
+
del calls[name]
|
|
170
|
+
return self
|
|
171
|
+
|
|
172
|
+
def difference(self, wrt_to: str=None):
|
|
173
|
+
"""Compute the difference w.r.t any algorithm by default is the best
|
|
174
|
+
|
|
175
|
+
>>> from sklearn.svm import LinearSVC
|
|
176
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
177
|
+
>>> from sklearn.datasets import load_iris
|
|
178
|
+
>>> from sklearn.model_selection import train_test_split
|
|
179
|
+
>>> from sklearn.base import clone
|
|
180
|
+
>>> from CompStats.interface import Perf
|
|
181
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
182
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
183
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
184
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
185
|
+
>>> hy = m.predict(X_val)
|
|
186
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
187
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
188
|
+
>>> perf.difference()
|
|
189
|
+
<Difference>
|
|
190
|
+
difference p-values w.r.t alg-1
|
|
191
|
+
forest 0.06
|
|
192
|
+
"""
|
|
193
|
+
if wrt_to is None:
|
|
194
|
+
wrt_to = self.best[0]
|
|
195
|
+
base = self.statistic_samples.calls[wrt_to]
|
|
196
|
+
sign = 1 if self.statistic_samples.BiB else -1
|
|
197
|
+
diff = dict()
|
|
198
|
+
for k, v in self.statistic_samples.calls.items():
|
|
199
|
+
if k == wrt_to:
|
|
200
|
+
continue
|
|
201
|
+
diff[k] = sign * (base - v)
|
|
202
|
+
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
203
|
+
statistic=self.statistic,
|
|
204
|
+
best=self.best[0])
|
|
205
|
+
diff_ins.statistic_samples.calls = diff
|
|
206
|
+
diff_ins.statistic_samples.info['best'] = self.best[0]
|
|
207
|
+
return diff_ins
|
|
208
|
+
|
|
209
|
+
@property
|
|
210
|
+
def best(self):
|
|
211
|
+
"""Best system"""
|
|
212
|
+
|
|
213
|
+
try:
|
|
214
|
+
return self._best
|
|
215
|
+
except AttributeError:
|
|
216
|
+
statistic = [(k, v) for k, v in self.statistic.items()]
|
|
217
|
+
statistic = sorted(statistic, key=lambda x: x[1],
|
|
218
|
+
reverse=self.statistic_samples.BiB)
|
|
219
|
+
self._best = statistic[0]
|
|
220
|
+
return self._best
|
|
221
|
+
|
|
222
|
+
@property
|
|
223
|
+
def statistic(self):
|
|
224
|
+
"""Statistic
|
|
225
|
+
|
|
226
|
+
>>> from sklearn.svm import LinearSVC
|
|
227
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
228
|
+
>>> from sklearn.datasets import load_iris
|
|
229
|
+
>>> from sklearn.model_selection import train_test_split
|
|
230
|
+
>>> from CompStats.interface import Perf
|
|
231
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
232
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
233
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
234
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
235
|
+
>>> hy = m.predict(X_val)
|
|
236
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
237
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
238
|
+
>>> perf.statistic
|
|
239
|
+
{'alg-1': 1.0, 'forest': 0.9500891265597148}
|
|
240
|
+
"""
|
|
241
|
+
|
|
242
|
+
data = sorted([(k, self.statistic_func(self.y_true, v))
|
|
243
|
+
for k, v in self.predictions.items()],
|
|
244
|
+
key=lambda x: x[1], reverse=self.statistic_samples.BiB)
|
|
245
|
+
return dict(data)
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def se(self):
|
|
249
|
+
"""Standard Error
|
|
250
|
+
|
|
251
|
+
>>> from sklearn.svm import LinearSVC
|
|
252
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
253
|
+
>>> from sklearn.datasets import load_iris
|
|
254
|
+
>>> from sklearn.model_selection import train_test_split
|
|
255
|
+
>>> from CompStats.interface import Perf
|
|
256
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
257
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
258
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
259
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
260
|
+
>>> hy = m.predict(X_val)
|
|
261
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
262
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
263
|
+
>>> perf.se
|
|
264
|
+
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
return SE(self.statistic_samples)
|
|
268
|
+
|
|
269
|
+
def plot(self, **kwargs):
|
|
270
|
+
"""plot with seaborn
|
|
271
|
+
|
|
272
|
+
>>> from sklearn.svm import LinearSVC
|
|
273
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
274
|
+
>>> from sklearn.datasets import load_iris
|
|
275
|
+
>>> from sklearn.model_selection import train_test_split
|
|
276
|
+
>>> from CompStats.interface import Perf
|
|
277
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
278
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
279
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
280
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
281
|
+
>>> hy = m.predict(X_val)
|
|
282
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
283
|
+
>>> perf = Perf(y_val, hy, score_func=None,
|
|
284
|
+
error_func=lambda y, hy: (y != hy).mean(),
|
|
285
|
+
forest=ens.predict(X_val))
|
|
286
|
+
>>> perf.plot()
|
|
287
|
+
"""
|
|
288
|
+
if self.score_func is not None:
|
|
289
|
+
value_name = 'Score'
|
|
290
|
+
else:
|
|
291
|
+
value_name = 'Error'
|
|
292
|
+
_ = dict(value_name=value_name)
|
|
293
|
+
_.update(kwargs)
|
|
294
|
+
return plot_performance(self.statistic_samples, **_)
|
|
295
|
+
|
|
296
|
+
@property
|
|
297
|
+
def n_jobs(self):
|
|
298
|
+
"""Number of jobs to compute the statistics"""
|
|
299
|
+
return self._n_jobs
|
|
300
|
+
|
|
301
|
+
@n_jobs.setter
|
|
302
|
+
def n_jobs(self, value):
|
|
303
|
+
self._n_jobs = value
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def statistic_func(self):
|
|
307
|
+
"""Statistic function"""
|
|
308
|
+
if self.score_func is not None:
|
|
309
|
+
return self.score_func
|
|
310
|
+
return self.error_func
|
|
311
|
+
|
|
312
|
+
@property
|
|
313
|
+
def statistic_samples(self):
|
|
314
|
+
"""Statistic Samples"""
|
|
315
|
+
|
|
316
|
+
samples = self._statistic_samples
|
|
317
|
+
algs = set(samples.calls.keys())
|
|
318
|
+
algs = set(self.predictions.keys()) - algs
|
|
319
|
+
if len(algs):
|
|
320
|
+
for key in progress_bar(algs, use_tqdm=self.use_tqdm):
|
|
321
|
+
samples(self.y_true, self.predictions[key], name=key)
|
|
322
|
+
return self._statistic_samples
|
|
323
|
+
|
|
324
|
+
@statistic_samples.setter
|
|
325
|
+
def statistic_samples(self, value):
|
|
326
|
+
self._statistic_samples = value
|
|
327
|
+
|
|
328
|
+
@property
|
|
329
|
+
def num_samples(self):
|
|
330
|
+
"""Number of bootstrap samples"""
|
|
331
|
+
return self._num_samples
|
|
332
|
+
|
|
333
|
+
@num_samples.setter
|
|
334
|
+
def num_samples(self, value):
|
|
335
|
+
self._num_samples = value
|
|
336
|
+
|
|
337
|
+
@property
|
|
338
|
+
def predictions(self):
|
|
339
|
+
"""Predictions"""
|
|
340
|
+
return self._predictions
|
|
341
|
+
|
|
342
|
+
@predictions.setter
|
|
343
|
+
def predictions(self, value):
|
|
344
|
+
self._predictions = value
|
|
345
|
+
|
|
346
|
+
@property
|
|
347
|
+
def y_true(self):
|
|
348
|
+
"""True output, gold standard o :math:`y`"""
|
|
349
|
+
|
|
350
|
+
return self._y_true
|
|
351
|
+
|
|
352
|
+
@y_true.setter
|
|
353
|
+
def y_true(self, value):
|
|
354
|
+
if isinstance(value, pd.DataFrame):
|
|
355
|
+
self._y_true = value['y'].to_numpy()
|
|
356
|
+
algs = {}
|
|
357
|
+
for c in value.columns:
|
|
358
|
+
if c == 'y':
|
|
359
|
+
continue
|
|
360
|
+
algs[c] = value[c].to_numpy()
|
|
361
|
+
self.predictions.update(algs)
|
|
362
|
+
return
|
|
363
|
+
self._y_true = value
|
|
364
|
+
|
|
365
|
+
@property
|
|
366
|
+
def score_func(self):
|
|
367
|
+
"""Score function"""
|
|
368
|
+
return self._score_func
|
|
369
|
+
|
|
370
|
+
@score_func.setter
|
|
371
|
+
def score_func(self, value):
|
|
372
|
+
self._score_func = value
|
|
373
|
+
if value is not None:
|
|
374
|
+
self.error_func = None
|
|
375
|
+
if hasattr(self, '_statistic_samples'):
|
|
376
|
+
self._statistic_samples.statistic = value
|
|
377
|
+
self._statistic_samples.BiB = True
|
|
378
|
+
|
|
379
|
+
@property
|
|
380
|
+
def error_func(self):
|
|
381
|
+
"""Error function"""
|
|
382
|
+
return self._error_func
|
|
383
|
+
|
|
384
|
+
@error_func.setter
|
|
385
|
+
def error_func(self, value):
|
|
386
|
+
self._error_func = value
|
|
387
|
+
if value is not None:
|
|
388
|
+
self.score_func = None
|
|
389
|
+
if hasattr(self, '_statistic_samples'):
|
|
390
|
+
self._statistic_samples.statistic = value
|
|
391
|
+
self._statistic_samples.BiB = False
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
@dataclass
|
|
395
|
+
class Difference:
|
|
396
|
+
"""Difference
|
|
397
|
+
|
|
398
|
+
>>> from sklearn.svm import LinearSVC
|
|
399
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
400
|
+
>>> from sklearn.datasets import load_iris
|
|
401
|
+
>>> from sklearn.model_selection import train_test_split
|
|
402
|
+
>>> from sklearn.base import clone
|
|
403
|
+
>>> from CompStats.interface import Perf
|
|
404
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
405
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
406
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
407
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
408
|
+
>>> hy = m.predict(X_val)
|
|
409
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
410
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
411
|
+
>>> diff = perf.difference()
|
|
412
|
+
>>> diff
|
|
413
|
+
<Difference>
|
|
414
|
+
difference p-values w.r.t alg-1
|
|
415
|
+
0.0780 <= forest
|
|
416
|
+
"""
|
|
417
|
+
|
|
418
|
+
statistic_samples:StatisticSamples=None
|
|
419
|
+
best:str=None
|
|
420
|
+
statistic:dict=None
|
|
421
|
+
|
|
422
|
+
def __repr__(self):
|
|
423
|
+
"""p-value"""
|
|
424
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
425
|
+
|
|
426
|
+
def __str__(self):
|
|
427
|
+
"""p-value"""
|
|
428
|
+
output = [f"difference p-values w.r.t {self.best}"]
|
|
429
|
+
for k, v in self.p_value().items():
|
|
430
|
+
output.append(f'{v:0.4f} <= {k}')
|
|
431
|
+
return "\n".join(output)
|
|
432
|
+
|
|
433
|
+
def p_value(self):
|
|
434
|
+
"""Compute p_value of the differences
|
|
435
|
+
|
|
436
|
+
>>> from sklearn.svm import LinearSVC
|
|
437
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
438
|
+
>>> from sklearn.datasets import load_iris
|
|
439
|
+
>>> from sklearn.model_selection import train_test_split
|
|
440
|
+
>>> from sklearn.base import clone
|
|
441
|
+
>>> from CompStats.interface import Perf
|
|
442
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
443
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
444
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
445
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
446
|
+
>>> hy = m.predict(X_val)
|
|
447
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
448
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
449
|
+
>>> diff = perf.difference()
|
|
450
|
+
>>> diff.p_value()
|
|
451
|
+
{'forest': np.float64(0.3)}
|
|
452
|
+
"""
|
|
453
|
+
values = []
|
|
454
|
+
sign = 1 if self.statistic_samples.BiB else -1
|
|
455
|
+
for k, v in self.statistic_samples.calls.items():
|
|
456
|
+
delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
|
|
457
|
+
values.append((k, (v > delta).mean()))
|
|
458
|
+
values.sort(key=lambda x: x[1])
|
|
459
|
+
return dict(values)
|
|
460
|
+
|
|
461
|
+
def plot(self, **kwargs):
|
|
462
|
+
"""Plot
|
|
463
|
+
|
|
464
|
+
>>> from sklearn.svm import LinearSVC
|
|
465
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
466
|
+
>>> from sklearn.datasets import load_iris
|
|
467
|
+
>>> from sklearn.model_selection import train_test_split
|
|
468
|
+
>>> from sklearn.base import clone
|
|
469
|
+
>>> from CompStats.interface import Perf
|
|
470
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
471
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
472
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
473
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
474
|
+
>>> hy = m.predict(X_val)
|
|
475
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
476
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
477
|
+
>>> diff = perf.difference()
|
|
478
|
+
>>> diff.plot()
|
|
479
|
+
"""
|
|
480
|
+
|
|
481
|
+
return plot_difference(self.statistic_samples, **kwargs)
|