CompStats 0.1.6__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/__init__.py +1 -1
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/bootstrap.py +4 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/interface.py +84 -7
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_interface.py +74 -1
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/utils.py +26 -1
- {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/PKG-INFO +10 -13
- {compstats-0.1.6 → compstats-0.1.7}/PKG-INFO +10 -13
- {compstats-0.1.6 → compstats-0.1.7}/README.rst +9 -12
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/measurements.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/metrics.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/performance.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_metrics.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/LICENSE +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/MANIFEST.in +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/pyproject.toml +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/setup.cfg +0 -0
- {compstats-0.1.6 → compstats-0.1.7}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.7'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -18,8 +18,10 @@ import pandas as pd
|
|
|
18
18
|
import numpy as np
|
|
19
19
|
from CompStats.bootstrap import StatisticSamples
|
|
20
20
|
from CompStats.utils import progress_bar
|
|
21
|
+
from CompStats import measurements
|
|
21
22
|
from CompStats.measurements import SE
|
|
22
23
|
from CompStats.performance import plot_performance, plot_difference
|
|
24
|
+
from CompStats.utils import dataframe
|
|
23
25
|
|
|
24
26
|
|
|
25
27
|
class Perf(object):
|
|
@@ -239,6 +241,13 @@ class Perf(object):
|
|
|
239
241
|
"""System with best performance"""
|
|
240
242
|
if hasattr(self, '_best') and self._best is not None:
|
|
241
243
|
return self._best
|
|
244
|
+
if not isinstance(self.statistic, dict):
|
|
245
|
+
key, value = list(self.statistic_samples.calls.items())[0]
|
|
246
|
+
if value.ndim == 1:
|
|
247
|
+
self._best = key
|
|
248
|
+
else:
|
|
249
|
+
self._best = np.array([key] * value.shape[1])
|
|
250
|
+
return self._best
|
|
242
251
|
BiB = True if self.statistic_samples.BiB else False
|
|
243
252
|
keys = np.array(list(self.statistic.keys()))
|
|
244
253
|
data = np.asanyarray([self.statistic[k]
|
|
@@ -322,7 +331,14 @@ class Perf(object):
|
|
|
322
331
|
return list(output.values())[0]
|
|
323
332
|
return output
|
|
324
333
|
|
|
325
|
-
def plot(self,
|
|
334
|
+
def plot(self, value_name:str=None,
|
|
335
|
+
var_name:str='Performance',
|
|
336
|
+
alg_legend:str='Algorithm',
|
|
337
|
+
perf_names:list=None,
|
|
338
|
+
CI:float=0.05,
|
|
339
|
+
kind:str='point', linestyle:str='none',
|
|
340
|
+
col_wrap:int=3, capsize:float=0.2,
|
|
341
|
+
**kwargs):
|
|
326
342
|
"""plot with seaborn
|
|
327
343
|
|
|
328
344
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -341,13 +357,37 @@ class Perf(object):
|
|
|
341
357
|
forest=ens.predict(X_val))
|
|
342
358
|
>>> perf.plot()
|
|
343
359
|
"""
|
|
360
|
+
import seaborn as sns
|
|
344
361
|
if self.score_func is not None:
|
|
345
362
|
value_name = 'Score'
|
|
346
363
|
else:
|
|
347
364
|
value_name = 'Error'
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
365
|
+
df = self.dataframe(value_name=value_name, var_name=var_name,
|
|
366
|
+
alg_legend=alg_legend, perf_names=perf_names)
|
|
367
|
+
if var_name not in df.columns:
|
|
368
|
+
var_name = None
|
|
369
|
+
col_wrap = None
|
|
370
|
+
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
371
|
+
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
372
|
+
y=alg_legend, col=var_name,
|
|
373
|
+
kind=kind, linestyle=linestyle,
|
|
374
|
+
col_wrap=col_wrap, capsize=capsize, **kwargs)
|
|
375
|
+
return f_grid
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def dataframe(self, value_name:str='Score',
|
|
379
|
+
var_name:str='Performance',
|
|
380
|
+
alg_legend:str='Algorithm',
|
|
381
|
+
perf_names:str=None):
|
|
382
|
+
"""Dataframe"""
|
|
383
|
+
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
384
|
+
func_name = self.statistic_func.__name__
|
|
385
|
+
perf_names = [f'{func_name}({i})'
|
|
386
|
+
for i, k in enumerate(self.best)]
|
|
387
|
+
return dataframe(self, value_name=value_name,
|
|
388
|
+
var_name=var_name,
|
|
389
|
+
alg_legend=alg_legend,
|
|
390
|
+
perf_names=perf_names)
|
|
351
391
|
|
|
352
392
|
@property
|
|
353
393
|
def n_jobs(self):
|
|
@@ -561,7 +601,28 @@ class Difference:
|
|
|
561
601
|
values.sort(key=lambda x: self.sorting_func(x[1]))
|
|
562
602
|
return dict(values)
|
|
563
603
|
|
|
564
|
-
def
|
|
604
|
+
def dataframe(self, value_name:str='Score',
|
|
605
|
+
var_name:str='Best',
|
|
606
|
+
alg_legend:str='Algorithm',
|
|
607
|
+
perf_names:str=None):
|
|
608
|
+
"""Dataframe"""
|
|
609
|
+
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
610
|
+
perf_names = [f'{alg}({k})'
|
|
611
|
+
for k, alg in enumerate(self.best)]
|
|
612
|
+
return dataframe(self, value_name=value_name,
|
|
613
|
+
var_name=var_name,
|
|
614
|
+
alg_legend=alg_legend,
|
|
615
|
+
perf_names=perf_names)
|
|
616
|
+
|
|
617
|
+
def plot(self, value_name:str='Difference',
|
|
618
|
+
var_name:str='Best',
|
|
619
|
+
alg_legend:str='Algorithm',
|
|
620
|
+
perf_names:list=None,
|
|
621
|
+
CI:float=0.05,
|
|
622
|
+
kind:str='point', linestyle:str='none',
|
|
623
|
+
col_wrap:int=3, capsize:float=0.2,
|
|
624
|
+
set_refline:bool=True,
|
|
625
|
+
**kwargs):
|
|
565
626
|
"""Plot
|
|
566
627
|
|
|
567
628
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -580,5 +641,21 @@ class Difference:
|
|
|
580
641
|
>>> diff = perf.difference()
|
|
581
642
|
>>> diff.plot()
|
|
582
643
|
"""
|
|
583
|
-
|
|
584
|
-
|
|
644
|
+
import seaborn as sns
|
|
645
|
+
df = self.dataframe(value_name=value_name,
|
|
646
|
+
var_name=var_name,
|
|
647
|
+
alg_legend=alg_legend, perf_names=perf_names)
|
|
648
|
+
title = var_name
|
|
649
|
+
if var_name not in df.columns:
|
|
650
|
+
var_name = None
|
|
651
|
+
col_wrap = None
|
|
652
|
+
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
653
|
+
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
654
|
+
y=alg_legend, col=var_name,
|
|
655
|
+
kind=kind, linestyle=linestyle,
|
|
656
|
+
col_wrap=col_wrap, capsize=capsize, **kwargs)
|
|
657
|
+
if set_refline:
|
|
658
|
+
f_grid.refline(x=0)
|
|
659
|
+
if isinstance(self.best, str):
|
|
660
|
+
f_grid.facet_axis(0, 0).set_title(f'{title} = {self.best}')
|
|
661
|
+
return f_grid
|
|
@@ -23,6 +23,79 @@ import pandas as pd
|
|
|
23
23
|
from CompStats.tests.test_performance import DATA
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
def test_Difference_dataframe():
|
|
27
|
+
"""Test Difference dataframe"""
|
|
28
|
+
from CompStats.metrics import f1_score
|
|
29
|
+
|
|
30
|
+
X, y = load_digits(return_X_y=True)
|
|
31
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
32
|
+
X_train, X_val, y_train, y_val = _
|
|
33
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
34
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
35
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
36
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
37
|
+
average=None,
|
|
38
|
+
num_samples=50)
|
|
39
|
+
score(nb.predict(X_val))
|
|
40
|
+
score(svm.predict(X_val))
|
|
41
|
+
diff = score.difference()
|
|
42
|
+
df = diff.dataframe()
|
|
43
|
+
assert 'Best' in df.columns
|
|
44
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
45
|
+
average='macro',
|
|
46
|
+
num_samples=50)
|
|
47
|
+
score(nb.predict(X_val))
|
|
48
|
+
score(svm.predict(X_val))
|
|
49
|
+
diff = score.difference()
|
|
50
|
+
df = diff.dataframe()
|
|
51
|
+
assert 'Best' not in df.columns
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_Perf_dataframe():
|
|
55
|
+
"""Test Perf dataframe"""
|
|
56
|
+
from CompStats.metrics import f1_score
|
|
57
|
+
|
|
58
|
+
X, y = load_digits(return_X_y=True)
|
|
59
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
60
|
+
X_train, X_val, y_train, y_val = _
|
|
61
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
62
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
63
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
64
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
65
|
+
average=None,
|
|
66
|
+
num_samples=50)
|
|
67
|
+
df = score.dataframe()
|
|
68
|
+
score(nb.predict(X_val))
|
|
69
|
+
score(svm.predict(X_val))
|
|
70
|
+
df = score.dataframe()
|
|
71
|
+
assert 'Performance' in df.columns
|
|
72
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
73
|
+
average='macro',
|
|
74
|
+
num_samples=50)
|
|
75
|
+
score(nb.predict(X_val))
|
|
76
|
+
score(svm.predict(X_val))
|
|
77
|
+
df = score.dataframe()
|
|
78
|
+
assert 'Performance' not in df.columns
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_Perf_plot_multi():
|
|
82
|
+
"""Test Perf plot multiple"""
|
|
83
|
+
from CompStats.metrics import f1_score
|
|
84
|
+
|
|
85
|
+
X, y = load_digits(return_X_y=True)
|
|
86
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
87
|
+
X_train, X_val, y_train, y_val = _
|
|
88
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
89
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
90
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
91
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
92
|
+
average=None,
|
|
93
|
+
num_samples=50)
|
|
94
|
+
score(nb.predict(X_val))
|
|
95
|
+
score(svm.predict(X_val))
|
|
96
|
+
f_grid = score.plot()
|
|
97
|
+
assert f_grid is not None
|
|
98
|
+
|
|
26
99
|
def test_Perf_statistic_one():
|
|
27
100
|
"""Test Perf statistic one alg"""
|
|
28
101
|
from CompStats.metrics import f1_score
|
|
@@ -208,7 +281,7 @@ def test_Difference_plot():
|
|
|
208
281
|
diff.plot()
|
|
209
282
|
|
|
210
283
|
|
|
211
|
-
def
|
|
284
|
+
def test_Perf_input_dataframe():
|
|
212
285
|
"""Test Perf with dataframe"""
|
|
213
286
|
from CompStats.interface import Perf
|
|
214
287
|
|
|
@@ -54,4 +54,29 @@ def metrics_docs(hy_name='y_pred', attr_name='score_func'):
|
|
|
54
54
|
return func(*args, **kwargs)
|
|
55
55
|
|
|
56
56
|
return inner
|
|
57
|
-
return perf_docs
|
|
57
|
+
return perf_docs
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def dataframe(instance, value_name:str='Score',
|
|
61
|
+
var_name:str='Performance',
|
|
62
|
+
alg_legend:str='Algorithm',
|
|
63
|
+
perf_names:list=None):
|
|
64
|
+
"""Dataframe"""
|
|
65
|
+
import pandas as pd
|
|
66
|
+
if isinstance(instance.best, str):
|
|
67
|
+
df = pd.DataFrame(dict(instance.statistic_samples.calls.items()))
|
|
68
|
+
return df.melt(var_name=alg_legend,
|
|
69
|
+
value_name=value_name)
|
|
70
|
+
df = pd.DataFrame()
|
|
71
|
+
if not isinstance(instance.statistic, dict):
|
|
72
|
+
iter = instance.statistic_samples.keys()
|
|
73
|
+
else:
|
|
74
|
+
iter = instance.statistic
|
|
75
|
+
for key in iter:
|
|
76
|
+
data = instance.statistic_samples[key]
|
|
77
|
+
_df = pd.DataFrame(data,
|
|
78
|
+
columns=perf_names).melt(value_name=value_name,
|
|
79
|
+
var_name=var_name)
|
|
80
|
+
_df[alg_legend] = key
|
|
81
|
+
df = pd.concat((df, _df))
|
|
82
|
+
return df
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.7
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -73,15 +73,12 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
73
73
|
|
|
74
74
|
>>> score = f1_score(y_val, hy, average='macro')
|
|
75
75
|
>>> score
|
|
76
|
-
<Perf(score_func=f1_score)>
|
|
77
|
-
Statistic with its standard error (se)
|
|
78
|
-
statistic (se)
|
|
79
|
-
0.9332 (0.0113) <= alg-1
|
|
76
|
+
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
80
77
|
|
|
81
78
|
The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
|
|
82
79
|
|
|
83
80
|
>>> score.statistic
|
|
84
|
-
|
|
81
|
+
0.9434834454375508
|
|
85
82
|
|
|
86
83
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
87
84
|
|
|
@@ -90,8 +87,8 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
90
87
|
<Perf(score_func=f1_score)>
|
|
91
88
|
Statistic with its standard error (se)
|
|
92
89
|
statistic (se)
|
|
93
|
-
0.
|
|
94
|
-
0.
|
|
90
|
+
0.9655 (0.0077) <= Random Forest
|
|
91
|
+
0.9435 (0.0099) <= alg-1
|
|
95
92
|
|
|
96
93
|
Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
|
|
97
94
|
|
|
@@ -100,18 +97,18 @@ Let us incorporate another prediction, now with the Naive Bayes classifier, as s
|
|
|
100
97
|
<Perf(score_func=f1_score)>
|
|
101
98
|
Statistic with its standard error (se)
|
|
102
99
|
statistic (se)
|
|
103
|
-
0.
|
|
104
|
-
0.
|
|
105
|
-
0.
|
|
100
|
+
0.9655 (0.0077) <= Random Forest
|
|
101
|
+
0.9435 (0.0099) <= alg-1
|
|
102
|
+
0.8549 (0.0153) <= Naive Bayes
|
|
106
103
|
|
|
107
104
|
The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
|
|
108
105
|
|
|
109
106
|
>>> diff = score.difference()
|
|
110
107
|
>>> diff
|
|
111
108
|
<Difference>
|
|
112
|
-
difference p-values
|
|
113
|
-
0.0000 <= alg-1
|
|
109
|
+
difference p-values w.r.t Random Forest
|
|
114
110
|
0.0000 <= Naive Bayes
|
|
111
|
+
0.0120 <= alg-1
|
|
115
112
|
|
|
116
113
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
117
114
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.7
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -73,15 +73,12 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
73
73
|
|
|
74
74
|
>>> score = f1_score(y_val, hy, average='macro')
|
|
75
75
|
>>> score
|
|
76
|
-
<Perf(score_func=f1_score)>
|
|
77
|
-
Statistic with its standard error (se)
|
|
78
|
-
statistic (se)
|
|
79
|
-
0.9332 (0.0113) <= alg-1
|
|
76
|
+
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
80
77
|
|
|
81
78
|
The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
|
|
82
79
|
|
|
83
80
|
>>> score.statistic
|
|
84
|
-
|
|
81
|
+
0.9434834454375508
|
|
85
82
|
|
|
86
83
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
87
84
|
|
|
@@ -90,8 +87,8 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
90
87
|
<Perf(score_func=f1_score)>
|
|
91
88
|
Statistic with its standard error (se)
|
|
92
89
|
statistic (se)
|
|
93
|
-
0.
|
|
94
|
-
0.
|
|
90
|
+
0.9655 (0.0077) <= Random Forest
|
|
91
|
+
0.9435 (0.0099) <= alg-1
|
|
95
92
|
|
|
96
93
|
Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
|
|
97
94
|
|
|
@@ -100,18 +97,18 @@ Let us incorporate another prediction, now with the Naive Bayes classifier, as s
|
|
|
100
97
|
<Perf(score_func=f1_score)>
|
|
101
98
|
Statistic with its standard error (se)
|
|
102
99
|
statistic (se)
|
|
103
|
-
0.
|
|
104
|
-
0.
|
|
105
|
-
0.
|
|
100
|
+
0.9655 (0.0077) <= Random Forest
|
|
101
|
+
0.9435 (0.0099) <= alg-1
|
|
102
|
+
0.8549 (0.0153) <= Naive Bayes
|
|
106
103
|
|
|
107
104
|
The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
|
|
108
105
|
|
|
109
106
|
>>> diff = score.difference()
|
|
110
107
|
>>> diff
|
|
111
108
|
<Difference>
|
|
112
|
-
difference p-values
|
|
113
|
-
0.0000 <= alg-1
|
|
109
|
+
difference p-values w.r.t Random Forest
|
|
114
110
|
0.0000 <= Naive Bayes
|
|
111
|
+
0.0120 <= alg-1
|
|
115
112
|
|
|
116
113
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
117
114
|
|
|
@@ -49,15 +49,12 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
49
49
|
|
|
50
50
|
>>> score = f1_score(y_val, hy, average='macro')
|
|
51
51
|
>>> score
|
|
52
|
-
<Perf(score_func=f1_score)>
|
|
53
|
-
Statistic with its standard error (se)
|
|
54
|
-
statistic (se)
|
|
55
|
-
0.9332 (0.0113) <= alg-1
|
|
52
|
+
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
56
53
|
|
|
57
54
|
The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
|
|
58
55
|
|
|
59
56
|
>>> score.statistic
|
|
60
|
-
|
|
57
|
+
0.9434834454375508
|
|
61
58
|
|
|
62
59
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
63
60
|
|
|
@@ -66,8 +63,8 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
66
63
|
<Perf(score_func=f1_score)>
|
|
67
64
|
Statistic with its standard error (se)
|
|
68
65
|
statistic (se)
|
|
69
|
-
0.
|
|
70
|
-
0.
|
|
66
|
+
0.9655 (0.0077) <= Random Forest
|
|
67
|
+
0.9435 (0.0099) <= alg-1
|
|
71
68
|
|
|
72
69
|
Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
|
|
73
70
|
|
|
@@ -76,18 +73,18 @@ Let us incorporate another prediction, now with the Naive Bayes classifier, as s
|
|
|
76
73
|
<Perf(score_func=f1_score)>
|
|
77
74
|
Statistic with its standard error (se)
|
|
78
75
|
statistic (se)
|
|
79
|
-
0.
|
|
80
|
-
0.
|
|
81
|
-
0.
|
|
76
|
+
0.9655 (0.0077) <= Random Forest
|
|
77
|
+
0.9435 (0.0099) <= alg-1
|
|
78
|
+
0.8549 (0.0153) <= Naive Bayes
|
|
82
79
|
|
|
83
80
|
The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
|
|
84
81
|
|
|
85
82
|
>>> diff = score.difference()
|
|
86
83
|
>>> diff
|
|
87
84
|
<Difference>
|
|
88
|
-
difference p-values
|
|
89
|
-
0.0000 <= alg-1
|
|
85
|
+
difference p-values w.r.t Random Forest
|
|
90
86
|
0.0000 <= Naive Bayes
|
|
87
|
+
0.0120 <= alg-1
|
|
91
88
|
|
|
92
89
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
93
90
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|