CompStats 0.1.5__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/__init__.py +1 -1
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/interface.py +79 -26
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_interface.py +99 -15
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_metrics.py +29 -76
- {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/PKG-INFO +20 -17
- {compstats-0.1.5 → compstats-0.1.6}/PKG-INFO +20 -17
- {compstats-0.1.5 → compstats-0.1.6}/README.rst +19 -16
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/measurements.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/metrics.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/performance.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats/utils.py +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/LICENSE +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/MANIFEST.in +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/pyproject.toml +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/setup.cfg +0 -0
- {compstats-0.1.5 → compstats-0.1.6}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.6'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -147,10 +147,21 @@ class Perf(object):
|
|
|
147
147
|
"""Prediction statistics with standard error in parenthesis"""
|
|
148
148
|
arg = 'score_func' if self.error_func is None else 'error_func'
|
|
149
149
|
func_name = self.statistic_func.__name__
|
|
150
|
-
|
|
151
|
-
|
|
150
|
+
statistic = self.statistic
|
|
151
|
+
if isinstance(statistic, dict):
|
|
152
|
+
return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
|
|
153
|
+
elif isinstance(statistic, float):
|
|
154
|
+
return f"<{self.__class__.__name__}({arg}={func_name}, statistic={statistic:0.4f}, se={self.se:0.4f})>"
|
|
155
|
+
desc = [f'{k:0.4f}' for k in statistic]
|
|
156
|
+
desc = ', '.join(desc)
|
|
157
|
+
desc_se = [f'{k:0.4f}' for k in self.se]
|
|
158
|
+
desc_se = ', '.join(desc_se)
|
|
159
|
+
return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
|
|
160
|
+
|
|
152
161
|
def __str__(self):
|
|
153
162
|
"""Prediction statistics with standard error in parenthesis"""
|
|
163
|
+
if not isinstance(self.statistic, dict):
|
|
164
|
+
return self.__repr__()
|
|
154
165
|
|
|
155
166
|
se = self.se
|
|
156
167
|
output = ["Statistic with its standard error (se)"]
|
|
@@ -173,6 +184,7 @@ class Perf(object):
|
|
|
173
184
|
if k == 0:
|
|
174
185
|
k = 1
|
|
175
186
|
name = f'alg-{k}'
|
|
187
|
+
self.best = None
|
|
176
188
|
self.predictions[name] = np.asanyarray(y_pred)
|
|
177
189
|
samples = self._statistic_samples
|
|
178
190
|
calls = samples.calls
|
|
@@ -180,7 +192,7 @@ class Perf(object):
|
|
|
180
192
|
del calls[name]
|
|
181
193
|
return self
|
|
182
194
|
|
|
183
|
-
def difference(self,
|
|
195
|
+
def difference(self, wrt: str=None):
|
|
184
196
|
"""Compute the difference w.r.t any algorithm by default is the best
|
|
185
197
|
|
|
186
198
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -201,37 +213,53 @@ class Perf(object):
|
|
|
201
213
|
difference p-values w.r.t alg-1
|
|
202
214
|
forest 0.06
|
|
203
215
|
"""
|
|
204
|
-
if
|
|
205
|
-
|
|
206
|
-
|
|
216
|
+
if wrt is None:
|
|
217
|
+
wrt = self.best
|
|
218
|
+
if isinstance(wrt, str):
|
|
219
|
+
base = self.statistic_samples.calls[wrt]
|
|
220
|
+
else:
|
|
221
|
+
base = np.array([self.statistic_samples.calls[key][:, col]
|
|
222
|
+
for col, key in enumerate(wrt)]).T
|
|
207
223
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
208
224
|
diff = dict()
|
|
209
225
|
for k, v in self.statistic_samples.calls.items():
|
|
210
|
-
if k ==
|
|
226
|
+
if base.ndim == 1 and k == wrt:
|
|
211
227
|
continue
|
|
212
228
|
diff[k] = sign * (base - v)
|
|
213
229
|
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
214
|
-
statistic=self.statistic
|
|
215
|
-
best=self.best[0])
|
|
230
|
+
statistic=self.statistic)
|
|
216
231
|
diff_ins.sorting_func = self.sorting_func
|
|
217
232
|
diff_ins.statistic_samples.calls = diff
|
|
218
|
-
diff_ins.statistic_samples.info['best'] = self.best
|
|
233
|
+
diff_ins.statistic_samples.info['best'] = self.best
|
|
234
|
+
diff_ins.best = self.best
|
|
219
235
|
return diff_ins
|
|
220
236
|
|
|
221
237
|
@property
|
|
222
238
|
def best(self):
|
|
223
|
-
"""
|
|
224
|
-
|
|
225
|
-
try:
|
|
239
|
+
"""System with best performance"""
|
|
240
|
+
if hasattr(self, '_best') and self._best is not None:
|
|
226
241
|
return self._best
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
242
|
+
BiB = True if self.statistic_samples.BiB else False
|
|
243
|
+
keys = np.array(list(self.statistic.keys()))
|
|
244
|
+
data = np.asanyarray([self.statistic[k]
|
|
245
|
+
for k in keys])
|
|
246
|
+
if isinstance(self.statistic[keys[0]], np.ndarray):
|
|
247
|
+
if BiB:
|
|
248
|
+
best = data.argmax(axis=0)
|
|
249
|
+
else:
|
|
250
|
+
best = data.argmin(axis=0)
|
|
251
|
+
else:
|
|
252
|
+
if BiB:
|
|
253
|
+
best = data.argmax()
|
|
254
|
+
else:
|
|
255
|
+
best = data.argmin()
|
|
256
|
+
self._best = keys[best]
|
|
233
257
|
return self._best
|
|
234
258
|
|
|
259
|
+
@best.setter
|
|
260
|
+
def best(self, value):
|
|
261
|
+
self._best = value
|
|
262
|
+
|
|
235
263
|
@property
|
|
236
264
|
def sorting_func(self):
|
|
237
265
|
"""Rank systems when multiple performances are used"""
|
|
@@ -265,6 +293,8 @@ class Perf(object):
|
|
|
265
293
|
for k, v in self.predictions.items()],
|
|
266
294
|
key=lambda x: self.sorting_func(x[1]),
|
|
267
295
|
reverse=self.statistic_samples.BiB)
|
|
296
|
+
if len(data) == 1:
|
|
297
|
+
return data[0][1]
|
|
268
298
|
return dict(data)
|
|
269
299
|
|
|
270
300
|
@property
|
|
@@ -287,7 +317,10 @@ class Perf(object):
|
|
|
287
317
|
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
288
318
|
"""
|
|
289
319
|
|
|
290
|
-
|
|
320
|
+
output = SE(self.statistic_samples)
|
|
321
|
+
if len(output) == 1:
|
|
322
|
+
return list(output.values())[0]
|
|
323
|
+
return output
|
|
291
324
|
|
|
292
325
|
def plot(self, **kwargs):
|
|
293
326
|
"""plot with seaborn
|
|
@@ -439,8 +472,8 @@ class Difference:
|
|
|
439
472
|
"""
|
|
440
473
|
|
|
441
474
|
statistic_samples:StatisticSamples=None
|
|
442
|
-
best:str=None
|
|
443
475
|
statistic:dict=None
|
|
476
|
+
best:str=None
|
|
444
477
|
|
|
445
478
|
@property
|
|
446
479
|
def sorting_func(self):
|
|
@@ -453,11 +486,19 @@ class Difference:
|
|
|
453
486
|
|
|
454
487
|
def __repr__(self):
|
|
455
488
|
"""p-value"""
|
|
456
|
-
return f"<{self.__class__.__name__}>\n{self}"
|
|
489
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
457
490
|
|
|
458
491
|
def __str__(self):
|
|
459
492
|
"""p-value"""
|
|
460
|
-
|
|
493
|
+
if isinstance(self.best, str):
|
|
494
|
+
best = f' w.r.t {self.best}'
|
|
495
|
+
else:
|
|
496
|
+
best = ''
|
|
497
|
+
output = [f"difference p-values {best}"]
|
|
498
|
+
best = self.best
|
|
499
|
+
if isinstance(best, np.ndarray):
|
|
500
|
+
desc = ', '.join(best)
|
|
501
|
+
output.append(f'{desc} <= Best')
|
|
461
502
|
for key, value in self.p_value().items():
|
|
462
503
|
if isinstance(value, float):
|
|
463
504
|
output.append(f'{value:0.4f} <= {key}')
|
|
@@ -465,8 +506,20 @@ class Difference:
|
|
|
465
506
|
desc = [f'{v:0.4f}' for v in value]
|
|
466
507
|
desc = ', '.join(desc)
|
|
467
508
|
desc = f'{desc} <= {key}'
|
|
509
|
+
output.append(desc)
|
|
468
510
|
return "\n".join(output)
|
|
469
511
|
|
|
512
|
+
def _delta_best(self):
|
|
513
|
+
"""Compute multiple delta"""
|
|
514
|
+
if isinstance(self.best, str):
|
|
515
|
+
return self.statistic[self.best]
|
|
516
|
+
keys = np.unique(self.best)
|
|
517
|
+
statistic = np.array([self.statistic[k]
|
|
518
|
+
for k in keys])
|
|
519
|
+
m = {v: k for k, v in enumerate(keys)}
|
|
520
|
+
best = np.array([m[x] for x in self.best])
|
|
521
|
+
return statistic[best, np.arange(best.shape[0])]
|
|
522
|
+
|
|
470
523
|
def p_value(self, right:bool=True):
|
|
471
524
|
"""Compute p_value of the differences
|
|
472
525
|
|
|
@@ -492,10 +545,10 @@ class Difference:
|
|
|
492
545
|
"""
|
|
493
546
|
values = []
|
|
494
547
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
495
|
-
|
|
548
|
+
delta_best = self._delta_best()
|
|
496
549
|
for k, v in self.statistic_samples.calls.items():
|
|
497
|
-
delta = 2 * sign * (
|
|
498
|
-
if
|
|
550
|
+
delta = 2 * sign * (delta_best - self.statistic[k])
|
|
551
|
+
if not isinstance(delta_best, np.ndarray):
|
|
499
552
|
if right:
|
|
500
553
|
values.append((k, (v >= delta).mean()))
|
|
501
554
|
else:
|
|
@@ -16,12 +16,109 @@ import numpy as np
|
|
|
16
16
|
from sklearn.base import clone
|
|
17
17
|
from sklearn.svm import LinearSVC
|
|
18
18
|
from sklearn.ensemble import RandomForestClassifier
|
|
19
|
-
from sklearn.
|
|
19
|
+
from sklearn.naive_bayes import GaussianNB
|
|
20
|
+
from sklearn.datasets import load_iris, load_digits
|
|
20
21
|
from sklearn.model_selection import train_test_split
|
|
21
22
|
import pandas as pd
|
|
22
23
|
from CompStats.tests.test_performance import DATA
|
|
23
24
|
|
|
24
25
|
|
|
26
|
+
def test_Perf_statistic_one():
|
|
27
|
+
"""Test Perf statistic one alg"""
|
|
28
|
+
from CompStats.metrics import f1_score
|
|
29
|
+
|
|
30
|
+
X, y = load_digits(return_X_y=True)
|
|
31
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
32
|
+
X_train, X_val, y_train, y_val = _
|
|
33
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
34
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
35
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
36
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
37
|
+
average=None,
|
|
38
|
+
num_samples=50)
|
|
39
|
+
assert isinstance(score.statistic, np.ndarray)
|
|
40
|
+
assert isinstance(str(score), str)
|
|
41
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
42
|
+
average='macro',
|
|
43
|
+
num_samples=50)
|
|
44
|
+
assert isinstance(score.statistic, float)
|
|
45
|
+
assert isinstance(str(score), str)
|
|
46
|
+
assert isinstance(score.se, float)
|
|
47
|
+
|
|
48
|
+
def test_Perf_best():
|
|
49
|
+
"""Test Perf best"""
|
|
50
|
+
from CompStats.metrics import f1_score
|
|
51
|
+
|
|
52
|
+
X, y = load_digits(return_X_y=True)
|
|
53
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
54
|
+
X_train, X_val, y_train, y_val = _
|
|
55
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
56
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
57
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
58
|
+
score = f1_score(y_val, average=None,
|
|
59
|
+
num_samples=50)
|
|
60
|
+
score(ens.predict(X_val), name='forest')
|
|
61
|
+
score(nb.predict(X_val), name='NB')
|
|
62
|
+
score(svm.predict(X_val), name='svm')
|
|
63
|
+
assert isinstance(score.best, np.ndarray)
|
|
64
|
+
score = f1_score(y_val, average='macro',
|
|
65
|
+
num_samples=50)
|
|
66
|
+
score(ens.predict(X_val), name='forest')
|
|
67
|
+
score(nb.predict(X_val), name='NB')
|
|
68
|
+
score(svm.predict(X_val), name='svm')
|
|
69
|
+
assert isinstance(score.best, str)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_difference_best():
|
|
73
|
+
"""Test multiple performance measures"""
|
|
74
|
+
from CompStats.metrics import f1_score
|
|
75
|
+
|
|
76
|
+
X, y = load_digits(return_X_y=True)
|
|
77
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
78
|
+
X_train, X_val, y_train, y_val = _
|
|
79
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
80
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
81
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
82
|
+
score = f1_score(y_val, average=None,
|
|
83
|
+
num_samples=50)
|
|
84
|
+
score(ens.predict(X_val), name='forest')
|
|
85
|
+
score(nb.predict(X_val), name='NB')
|
|
86
|
+
score(svm.predict(X_val), name='svm')
|
|
87
|
+
diff = score.difference()
|
|
88
|
+
assert isinstance(diff.best, np.ndarray)
|
|
89
|
+
score = f1_score(y_val, average='macro',
|
|
90
|
+
num_samples=50)
|
|
91
|
+
score(ens.predict(X_val), name='forest')
|
|
92
|
+
score(nb.predict(X_val), name='NB')
|
|
93
|
+
score(svm.predict(X_val), name='svm')
|
|
94
|
+
diff = score.difference()
|
|
95
|
+
assert isinstance(diff.best, str)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_difference_str__():
|
|
99
|
+
"""Test f1_score"""
|
|
100
|
+
from CompStats.metrics import f1_score
|
|
101
|
+
|
|
102
|
+
X, y = load_iris(return_X_y=True)
|
|
103
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
104
|
+
X_train, X_val, y_train, y_val = _
|
|
105
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
106
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
107
|
+
perf = f1_score(y_val, nb.predict(X_val),
|
|
108
|
+
forest=ens.predict(X_val),
|
|
109
|
+
num_samples=50, average=None)
|
|
110
|
+
diff = perf.difference()
|
|
111
|
+
p_values = diff.p_value(right=False)
|
|
112
|
+
dd = list(p_values.values())[0]
|
|
113
|
+
assert isinstance(dd, np.ndarray)
|
|
114
|
+
for average in ['macro', None]:
|
|
115
|
+
perf = f1_score(y_val, nb.predict(X_val),
|
|
116
|
+
forest=ens.predict(X_val),
|
|
117
|
+
num_samples=50, average=average)
|
|
118
|
+
diff = perf.difference()
|
|
119
|
+
print(diff)
|
|
120
|
+
|
|
121
|
+
|
|
25
122
|
def test_Perf():
|
|
26
123
|
"""Test perf"""
|
|
27
124
|
from CompStats.interface import Perf
|
|
@@ -47,6 +144,7 @@ def test_Perf_statistic():
|
|
|
47
144
|
X_train, X_val, y_train, y_val = _
|
|
48
145
|
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
49
146
|
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
147
|
+
perf(ens.predict(X_val))
|
|
50
148
|
assert 'forest' in perf.statistic
|
|
51
149
|
|
|
52
150
|
|
|
@@ -79,20 +177,6 @@ def test_Perf_clone():
|
|
|
79
177
|
assert np.all(samples == perf2.statistic_samples._samples)
|
|
80
178
|
|
|
81
179
|
|
|
82
|
-
def test_Perf_best():
|
|
83
|
-
"""Test Perf.best"""
|
|
84
|
-
from CompStats.interface import Perf
|
|
85
|
-
|
|
86
|
-
X, y = load_iris(return_X_y=True)
|
|
87
|
-
_ = train_test_split(X, y, test_size=0.3)
|
|
88
|
-
X_train, X_val, y_train, y_val = _
|
|
89
|
-
m = LinearSVC().fit(X_train, y_train)
|
|
90
|
-
hy = m.predict(X_val)
|
|
91
|
-
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
92
|
-
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
93
|
-
assert len(perf.best) == 2
|
|
94
|
-
|
|
95
|
-
|
|
96
180
|
def test_Perf_difference():
|
|
97
181
|
"""Test difference"""
|
|
98
182
|
from CompStats.interface import Perf, Difference
|
|
@@ -11,8 +11,6 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
import numpy as np
|
|
15
|
-
from sklearn.svm import LinearSVC
|
|
16
14
|
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
|
|
17
15
|
from sklearn.naive_bayes import GaussianNB
|
|
18
16
|
from sklearn.datasets import load_iris, load_diabetes
|
|
@@ -20,24 +18,6 @@ from sklearn.model_selection import train_test_split
|
|
|
20
18
|
from sklearn import metrics
|
|
21
19
|
|
|
22
20
|
|
|
23
|
-
def test_difference_f1_score():
|
|
24
|
-
"""Test f1_score"""
|
|
25
|
-
from CompStats.metrics import f1_score
|
|
26
|
-
|
|
27
|
-
X, y = load_iris(return_X_y=True)
|
|
28
|
-
_ = train_test_split(X, y, test_size=0.3)
|
|
29
|
-
X_train, X_val, y_train, y_val = _
|
|
30
|
-
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
31
|
-
nb = GaussianNB().fit(X_train, y_train)
|
|
32
|
-
perf = f1_score(y_val, nb.predict(X_val),
|
|
33
|
-
forest=ens.predict(X_val),
|
|
34
|
-
num_samples=50, average=None)
|
|
35
|
-
diff = perf.difference()
|
|
36
|
-
p_values = diff.p_value(right=False)
|
|
37
|
-
dd = list(p_values.values())[0]
|
|
38
|
-
assert isinstance(dd, np.ndarray)
|
|
39
|
-
print(diff)
|
|
40
|
-
|
|
41
21
|
def test_f1_score():
|
|
42
22
|
"""Test f1_score"""
|
|
43
23
|
from CompStats.metrics import f1_score
|
|
@@ -49,9 +29,9 @@ def test_f1_score():
|
|
|
49
29
|
hy = ens.predict(X_val)
|
|
50
30
|
perf = f1_score(y_val, forest=hy,
|
|
51
31
|
num_samples=50, average='macro')
|
|
52
|
-
assert
|
|
32
|
+
assert isinstance(perf.statistic, float)
|
|
53
33
|
_ = metrics.f1_score(y_val, hy, average='macro')
|
|
54
|
-
assert _ == perf.statistic
|
|
34
|
+
assert _ == perf.statistic
|
|
55
35
|
perf = f1_score(y_val, hy, average=None)
|
|
56
36
|
assert str(perf) is not None
|
|
57
37
|
nb = GaussianNB().fit(X_train, y_train)
|
|
@@ -70,9 +50,8 @@ def test_accuracy_score():
|
|
|
70
50
|
hy = ens.predict(X_val)
|
|
71
51
|
perf = accuracy_score(y_val, forest=hy,
|
|
72
52
|
num_samples=50)
|
|
73
|
-
assert 'forest' in perf.statistic
|
|
74
53
|
_ = metrics.accuracy_score(y_val, hy)
|
|
75
|
-
assert _ == perf.statistic
|
|
54
|
+
assert _ == perf.statistic
|
|
76
55
|
|
|
77
56
|
|
|
78
57
|
def test_balanced_accuracy_score():
|
|
@@ -86,9 +65,8 @@ def test_balanced_accuracy_score():
|
|
|
86
65
|
hy = ens.predict(X_val)
|
|
87
66
|
perf = balanced_accuracy_score(y_val, forest=hy,
|
|
88
67
|
num_samples=50)
|
|
89
|
-
assert 'forest' in perf.statistic
|
|
90
68
|
_ = metrics.balanced_accuracy_score(y_val, hy)
|
|
91
|
-
assert _ == perf.statistic
|
|
69
|
+
assert _ == perf.statistic
|
|
92
70
|
|
|
93
71
|
|
|
94
72
|
def test_top_k_accuracy_score():
|
|
@@ -103,9 +81,8 @@ def test_top_k_accuracy_score():
|
|
|
103
81
|
perf = top_k_accuracy_score(y_val,
|
|
104
82
|
forest=hy,
|
|
105
83
|
num_samples=50)
|
|
106
|
-
assert 'forest' in perf.statistic
|
|
107
84
|
_ = metrics.top_k_accuracy_score(y_val, hy)
|
|
108
|
-
assert _ == perf.statistic
|
|
85
|
+
assert _ == perf.statistic
|
|
109
86
|
|
|
110
87
|
|
|
111
88
|
def test_average_precision_score():
|
|
@@ -120,9 +97,8 @@ def test_average_precision_score():
|
|
|
120
97
|
perf = average_precision_score(y_val,
|
|
121
98
|
forest=hy,
|
|
122
99
|
num_samples=50)
|
|
123
|
-
assert 'forest' in perf.statistic
|
|
124
100
|
_ = metrics.average_precision_score(y_val, hy)
|
|
125
|
-
assert _ == perf.statistic
|
|
101
|
+
assert _ == perf.statistic
|
|
126
102
|
|
|
127
103
|
|
|
128
104
|
def test_brier_score_loss():
|
|
@@ -138,11 +114,8 @@ def test_brier_score_loss():
|
|
|
138
114
|
perf = brier_score_loss(np.where(y_val == 0, 1, 0),
|
|
139
115
|
forest=hy,
|
|
140
116
|
num_samples=50)
|
|
141
|
-
nb = GaussianNB().fit(X_train, y_train)
|
|
142
|
-
perf(nb.predict_proba(X_val)[:, 0])
|
|
143
|
-
assert 'forest' in perf.statistic
|
|
144
117
|
_ = metrics.brier_score_loss(np.where(y_val == 0, 1, 0), hy)
|
|
145
|
-
assert _ == perf.statistic
|
|
118
|
+
assert _ == perf.statistic
|
|
146
119
|
|
|
147
120
|
|
|
148
121
|
def test_log_loss():
|
|
@@ -158,11 +131,8 @@ def test_log_loss():
|
|
|
158
131
|
perf = log_loss(y_val,
|
|
159
132
|
forest=hy,
|
|
160
133
|
num_samples=50)
|
|
161
|
-
nb = GaussianNB().fit(X_train, y_train)
|
|
162
|
-
perf(nb.predict_proba(X_val))
|
|
163
|
-
assert 'forest' in perf.statistic
|
|
164
134
|
_ = metrics.log_loss(y_val, hy)
|
|
165
|
-
assert _ == perf.statistic
|
|
135
|
+
assert _ == perf.statistic
|
|
166
136
|
|
|
167
137
|
|
|
168
138
|
def test_precision_score():
|
|
@@ -178,9 +148,8 @@ def test_precision_score():
|
|
|
178
148
|
perf = precision_score(y_val,
|
|
179
149
|
forest=hy,
|
|
180
150
|
num_samples=50, average='macro')
|
|
181
|
-
assert 'forest' in perf.statistic
|
|
182
151
|
_ = metrics.precision_score(y_val, hy, average='macro')
|
|
183
|
-
assert _ == perf.statistic
|
|
152
|
+
assert _ == perf.statistic
|
|
184
153
|
|
|
185
154
|
|
|
186
155
|
def test_recall_score():
|
|
@@ -196,9 +165,8 @@ def test_recall_score():
|
|
|
196
165
|
perf = recall_score(y_val,
|
|
197
166
|
forest=hy,
|
|
198
167
|
num_samples=50, average='macro')
|
|
199
|
-
assert 'forest' in perf.statistic
|
|
200
168
|
_ = metrics.recall_score(y_val, hy, average='macro')
|
|
201
|
-
assert _ == perf.statistic
|
|
169
|
+
assert _ == perf.statistic
|
|
202
170
|
|
|
203
171
|
|
|
204
172
|
def test_jaccard_score():
|
|
@@ -214,9 +182,8 @@ def test_jaccard_score():
|
|
|
214
182
|
perf = jaccard_score(y_val,
|
|
215
183
|
forest=hy,
|
|
216
184
|
num_samples=50, average='macro')
|
|
217
|
-
assert 'forest' in perf.statistic
|
|
218
185
|
_ = metrics.jaccard_score(y_val, hy, average='macro')
|
|
219
|
-
assert _ == perf.statistic
|
|
186
|
+
assert _ == perf.statistic
|
|
220
187
|
|
|
221
188
|
|
|
222
189
|
def test_roc_auc_score():
|
|
@@ -232,10 +199,9 @@ def test_roc_auc_score():
|
|
|
232
199
|
perf = roc_auc_score(y_val,
|
|
233
200
|
forest=hy, multi_class='ovr',
|
|
234
201
|
num_samples=50, average='macro')
|
|
235
|
-
assert 'forest' in perf.statistic
|
|
236
202
|
_ = metrics.roc_auc_score(y_val, hy, multi_class='ovr',
|
|
237
203
|
average='macro')
|
|
238
|
-
assert _ == perf.statistic
|
|
204
|
+
assert _ == perf.statistic
|
|
239
205
|
|
|
240
206
|
|
|
241
207
|
def test_d2_log_loss_score():
|
|
@@ -251,9 +217,8 @@ def test_d2_log_loss_score():
|
|
|
251
217
|
perf = d2_log_loss_score(y_val,
|
|
252
218
|
forest=hy,
|
|
253
219
|
num_samples=50)
|
|
254
|
-
assert 'forest' in perf.statistic
|
|
255
220
|
_ = metrics.d2_log_loss_score(y_val, hy)
|
|
256
|
-
assert _ == perf.statistic
|
|
221
|
+
assert _ == perf.statistic
|
|
257
222
|
|
|
258
223
|
|
|
259
224
|
def test_explained_variance_score():
|
|
@@ -268,9 +233,8 @@ def test_explained_variance_score():
|
|
|
268
233
|
perf = explained_variance_score(y_val,
|
|
269
234
|
forest=hy,
|
|
270
235
|
num_samples=50)
|
|
271
|
-
assert 'forest' in perf.statistic
|
|
272
236
|
_ = metrics.explained_variance_score(y_val, hy)
|
|
273
|
-
assert _ == perf.statistic
|
|
237
|
+
assert _ == perf.statistic
|
|
274
238
|
|
|
275
239
|
|
|
276
240
|
def test_max_error():
|
|
@@ -285,9 +249,8 @@ def test_max_error():
|
|
|
285
249
|
perf = max_error(y_val,
|
|
286
250
|
forest=hy,
|
|
287
251
|
num_samples=50)
|
|
288
|
-
assert 'forest' in perf.statistic
|
|
289
252
|
_ = metrics.max_error(y_val, hy)
|
|
290
|
-
assert _ == perf.statistic
|
|
253
|
+
assert _ == perf.statistic
|
|
291
254
|
|
|
292
255
|
|
|
293
256
|
def test_mean_absolute_error():
|
|
@@ -302,9 +265,8 @@ def test_mean_absolute_error():
|
|
|
302
265
|
perf = mean_absolute_error(y_val,
|
|
303
266
|
forest=hy,
|
|
304
267
|
num_samples=50)
|
|
305
|
-
assert 'forest' in perf.statistic
|
|
306
268
|
_ = metrics.mean_absolute_error(y_val, hy)
|
|
307
|
-
assert _ == perf.statistic
|
|
269
|
+
assert _ == perf.statistic
|
|
308
270
|
|
|
309
271
|
|
|
310
272
|
def test_mean_squared_error():
|
|
@@ -319,9 +281,8 @@ def test_mean_squared_error():
|
|
|
319
281
|
perf = mean_squared_error(y_val,
|
|
320
282
|
forest=hy,
|
|
321
283
|
num_samples=50)
|
|
322
|
-
assert 'forest' in perf.statistic
|
|
323
284
|
_ = metrics.mean_squared_error(y_val, hy)
|
|
324
|
-
assert _ == perf.statistic
|
|
285
|
+
assert _ == perf.statistic
|
|
325
286
|
|
|
326
287
|
|
|
327
288
|
def test_root_mean_squared_error():
|
|
@@ -336,9 +297,8 @@ def test_root_mean_squared_error():
|
|
|
336
297
|
perf = root_mean_squared_error(y_val,
|
|
337
298
|
forest=hy,
|
|
338
299
|
num_samples=50)
|
|
339
|
-
assert 'forest' in perf.statistic
|
|
340
300
|
_ = metrics.root_mean_squared_error(y_val, hy)
|
|
341
|
-
assert _ == perf.statistic
|
|
301
|
+
assert _ == perf.statistic
|
|
342
302
|
|
|
343
303
|
|
|
344
304
|
def test_mean_squared_log_error():
|
|
@@ -353,9 +313,8 @@ def test_mean_squared_log_error():
|
|
|
353
313
|
perf = mean_squared_log_error(y_val,
|
|
354
314
|
forest=hy,
|
|
355
315
|
num_samples=50)
|
|
356
|
-
assert 'forest' in perf.statistic
|
|
357
316
|
_ = metrics.mean_squared_log_error(y_val, hy)
|
|
358
|
-
assert _ == perf.statistic
|
|
317
|
+
assert _ == perf.statistic
|
|
359
318
|
|
|
360
319
|
|
|
361
320
|
def test_root_mean_squared_log_error():
|
|
@@ -370,9 +329,8 @@ def test_root_mean_squared_log_error():
|
|
|
370
329
|
perf = root_mean_squared_log_error(y_val,
|
|
371
330
|
forest=hy,
|
|
372
331
|
num_samples=50)
|
|
373
|
-
assert 'forest' in perf.statistic
|
|
374
332
|
_ = metrics.root_mean_squared_log_error(y_val, hy)
|
|
375
|
-
assert _ == perf.statistic
|
|
333
|
+
assert _ == perf.statistic
|
|
376
334
|
|
|
377
335
|
|
|
378
336
|
def test_median_absolute_error():
|
|
@@ -387,9 +345,8 @@ def test_median_absolute_error():
|
|
|
387
345
|
perf = median_absolute_error(y_val,
|
|
388
346
|
forest=hy,
|
|
389
347
|
num_samples=50)
|
|
390
|
-
assert 'forest' in perf.statistic
|
|
391
348
|
_ = metrics.median_absolute_error(y_val, hy)
|
|
392
|
-
assert _ == perf.statistic
|
|
349
|
+
assert _ == perf.statistic
|
|
393
350
|
|
|
394
351
|
|
|
395
352
|
def test_r2_score():
|
|
@@ -402,11 +359,11 @@ def test_r2_score():
|
|
|
402
359
|
ens = RandomForestRegressor().fit(X_train, y_train)
|
|
403
360
|
hy = ens.predict(X_val)
|
|
404
361
|
perf = r2_score(y_val,
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
362
|
+
forest=hy,
|
|
363
|
+
num_samples=50)
|
|
364
|
+
|
|
408
365
|
_ = metrics.r2_score(y_val, hy)
|
|
409
|
-
assert _ == perf.statistic
|
|
366
|
+
assert _ == perf.statistic
|
|
410
367
|
|
|
411
368
|
|
|
412
369
|
def test_mean_poisson_deviance():
|
|
@@ -421,9 +378,8 @@ def test_mean_poisson_deviance():
|
|
|
421
378
|
perf = mean_poisson_deviance(y_val,
|
|
422
379
|
forest=hy,
|
|
423
380
|
num_samples=50)
|
|
424
|
-
assert 'forest' in perf.statistic
|
|
425
381
|
_ = metrics.mean_poisson_deviance(y_val, hy)
|
|
426
|
-
assert _ == perf.statistic
|
|
382
|
+
assert _ == perf.statistic
|
|
427
383
|
|
|
428
384
|
|
|
429
385
|
def test_mean_gamma_deviance():
|
|
@@ -438,9 +394,8 @@ def test_mean_gamma_deviance():
|
|
|
438
394
|
perf = mean_gamma_deviance(y_val,
|
|
439
395
|
forest=hy,
|
|
440
396
|
num_samples=50)
|
|
441
|
-
assert 'forest' in perf.statistic
|
|
442
397
|
_ = metrics.mean_gamma_deviance(y_val, hy)
|
|
443
|
-
assert _ == perf.statistic
|
|
398
|
+
assert _ == perf.statistic
|
|
444
399
|
|
|
445
400
|
|
|
446
401
|
def test_mean_absolute_percentage_error():
|
|
@@ -455,9 +410,8 @@ def test_mean_absolute_percentage_error():
|
|
|
455
410
|
perf = mean_absolute_percentage_error(y_val,
|
|
456
411
|
forest=hy,
|
|
457
412
|
num_samples=50)
|
|
458
|
-
assert 'forest' in perf.statistic
|
|
459
413
|
_ = metrics.mean_absolute_percentage_error(y_val, hy)
|
|
460
|
-
assert _ == perf.statistic
|
|
414
|
+
assert _ == perf.statistic
|
|
461
415
|
|
|
462
416
|
|
|
463
417
|
def test_d2_absolute_error_score():
|
|
@@ -472,6 +426,5 @@ def test_d2_absolute_error_score():
|
|
|
472
426
|
perf = d2_absolute_error_score(y_val,
|
|
473
427
|
forest=hy,
|
|
474
428
|
num_samples=50)
|
|
475
|
-
assert 'forest' in perf.statistic
|
|
476
429
|
_ = metrics.d2_absolute_error_score(y_val, hy)
|
|
477
|
-
assert _ == perf.statistic
|
|
430
|
+
assert _ == perf.statistic
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -73,33 +73,36 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
73
73
|
|
|
74
74
|
>>> score = f1_score(y_val, hy, average='macro')
|
|
75
75
|
>>> score
|
|
76
|
-
<Perf>
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
<Perf(score_func=f1_score)>
|
|
77
|
+
Statistic with its standard error (se)
|
|
78
|
+
statistic (se)
|
|
79
|
+
0.9332 (0.0113) <= alg-1
|
|
79
80
|
|
|
80
81
|
The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
|
|
81
82
|
|
|
82
|
-
>>> score.statistic
|
|
83
|
-
{'alg-1': 0.
|
|
83
|
+
>>> score.statistic
|
|
84
|
+
{'alg-1': 0.9332035615949114}
|
|
84
85
|
|
|
85
86
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
86
87
|
|
|
87
88
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
88
89
|
>>> score(ens.predict(X_val), name='Random Forest')
|
|
89
|
-
<Perf>
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
<Perf(score_func=f1_score)>
|
|
91
|
+
Statistic with its standard error (se)
|
|
92
|
+
statistic (se)
|
|
93
|
+
0.9756 (0.0061) <= Random Forest
|
|
94
|
+
0.9332 (0.0113) <= alg-1
|
|
93
95
|
|
|
94
96
|
Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
|
|
95
97
|
|
|
96
98
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
97
99
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
98
|
-
<Perf>
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
<Perf(score_func=f1_score)>
|
|
101
|
+
Statistic with its standard error (se)
|
|
102
|
+
statistic (se)
|
|
103
|
+
0.9756 (0.0061) <= Random Forest
|
|
104
|
+
0.9332 (0.0113) <= alg-1
|
|
105
|
+
0.8198 (0.0144) <= Naive Bayes
|
|
103
106
|
|
|
104
107
|
The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
|
|
105
108
|
|
|
@@ -107,8 +110,8 @@ The final step is to compare the performance of the three classifiers, which can
|
|
|
107
110
|
>>> diff
|
|
108
111
|
<Difference>
|
|
109
112
|
difference p-values w.r.t Random Forest
|
|
110
|
-
alg-1
|
|
111
|
-
Naive Bayes
|
|
113
|
+
0.0000 <= alg-1
|
|
114
|
+
0.0000 <= Naive Bayes
|
|
112
115
|
|
|
113
116
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
114
117
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.6
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -73,33 +73,36 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
73
73
|
|
|
74
74
|
>>> score = f1_score(y_val, hy, average='macro')
|
|
75
75
|
>>> score
|
|
76
|
-
<Perf>
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
<Perf(score_func=f1_score)>
|
|
77
|
+
Statistic with its standard error (se)
|
|
78
|
+
statistic (se)
|
|
79
|
+
0.9332 (0.0113) <= alg-1
|
|
79
80
|
|
|
80
81
|
The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
|
|
81
82
|
|
|
82
|
-
>>> score.statistic
|
|
83
|
-
{'alg-1': 0.
|
|
83
|
+
>>> score.statistic
|
|
84
|
+
{'alg-1': 0.9332035615949114}
|
|
84
85
|
|
|
85
86
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
86
87
|
|
|
87
88
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
88
89
|
>>> score(ens.predict(X_val), name='Random Forest')
|
|
89
|
-
<Perf>
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
<Perf(score_func=f1_score)>
|
|
91
|
+
Statistic with its standard error (se)
|
|
92
|
+
statistic (se)
|
|
93
|
+
0.9756 (0.0061) <= Random Forest
|
|
94
|
+
0.9332 (0.0113) <= alg-1
|
|
93
95
|
|
|
94
96
|
Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
|
|
95
97
|
|
|
96
98
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
97
99
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
98
|
-
<Perf>
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
100
|
+
<Perf(score_func=f1_score)>
|
|
101
|
+
Statistic with its standard error (se)
|
|
102
|
+
statistic (se)
|
|
103
|
+
0.9756 (0.0061) <= Random Forest
|
|
104
|
+
0.9332 (0.0113) <= alg-1
|
|
105
|
+
0.8198 (0.0144) <= Naive Bayes
|
|
103
106
|
|
|
104
107
|
The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
|
|
105
108
|
|
|
@@ -107,8 +110,8 @@ The final step is to compare the performance of the three classifiers, which can
|
|
|
107
110
|
>>> diff
|
|
108
111
|
<Difference>
|
|
109
112
|
difference p-values w.r.t Random Forest
|
|
110
|
-
alg-1
|
|
111
|
-
Naive Bayes
|
|
113
|
+
0.0000 <= alg-1
|
|
114
|
+
0.0000 <= Naive Bayes
|
|
112
115
|
|
|
113
116
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
114
117
|
|
|
@@ -49,33 +49,36 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
49
49
|
|
|
50
50
|
>>> score = f1_score(y_val, hy, average='macro')
|
|
51
51
|
>>> score
|
|
52
|
-
<Perf>
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
<Perf(score_func=f1_score)>
|
|
53
|
+
Statistic with its standard error (se)
|
|
54
|
+
statistic (se)
|
|
55
|
+
0.9332 (0.0113) <= alg-1
|
|
55
56
|
|
|
56
57
|
The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
|
|
57
58
|
|
|
58
|
-
>>> score.statistic
|
|
59
|
-
{'alg-1': 0.
|
|
59
|
+
>>> score.statistic
|
|
60
|
+
{'alg-1': 0.9332035615949114}
|
|
60
61
|
|
|
61
62
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
62
63
|
|
|
63
64
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
64
65
|
>>> score(ens.predict(X_val), name='Random Forest')
|
|
65
|
-
<Perf>
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
66
|
+
<Perf(score_func=f1_score)>
|
|
67
|
+
Statistic with its standard error (se)
|
|
68
|
+
statistic (se)
|
|
69
|
+
0.9756 (0.0061) <= Random Forest
|
|
70
|
+
0.9332 (0.0113) <= alg-1
|
|
69
71
|
|
|
70
72
|
Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
|
|
71
73
|
|
|
72
74
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
73
75
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
74
|
-
<Perf>
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
<Perf(score_func=f1_score)>
|
|
77
|
+
Statistic with its standard error (se)
|
|
78
|
+
statistic (se)
|
|
79
|
+
0.9756 (0.0061) <= Random Forest
|
|
80
|
+
0.9332 (0.0113) <= alg-1
|
|
81
|
+
0.8198 (0.0144) <= Naive Bayes
|
|
79
82
|
|
|
80
83
|
The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
|
|
81
84
|
|
|
@@ -83,8 +86,8 @@ The final step is to compare the performance of the three classifiers, which can
|
|
|
83
86
|
>>> diff
|
|
84
87
|
<Difference>
|
|
85
88
|
difference p-values w.r.t Random Forest
|
|
86
|
-
alg-1
|
|
87
|
-
Naive Bayes
|
|
89
|
+
0.0000 <= alg-1
|
|
90
|
+
0.0000 <= Naive Bayes
|
|
88
91
|
|
|
89
92
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
90
93
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|