CompStats 0.1.9__tar.gz → 0.1.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/__init__.py +1 -1
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/interface.py +67 -14
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_interface.py +19 -1
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/utils.py +8 -5
- {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/PKG-INFO +22 -14
- {compstats-0.1.9 → compstats-0.1.11}/PKG-INFO +22 -14
- {compstats-0.1.9 → compstats-0.1.11}/README.rst +21 -13
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/measurements.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/metrics.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/performance.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_metrics.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/LICENSE +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/MANIFEST.in +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/pyproject.toml +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/setup.cfg +0 -0
- {compstats-0.1.9 → compstats-0.1.11}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.11'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -20,7 +20,6 @@ from CompStats.bootstrap import StatisticSamples
|
|
|
20
20
|
from CompStats.utils import progress_bar
|
|
21
21
|
from CompStats import measurements
|
|
22
22
|
from CompStats.measurements import SE
|
|
23
|
-
from CompStats.performance import plot_performance, plot_difference
|
|
24
23
|
from CompStats.utils import dataframe
|
|
25
24
|
|
|
26
25
|
|
|
@@ -159,7 +158,7 @@ class Perf(object):
|
|
|
159
158
|
desc_se = [f'{k:0.4f}' for k in self.se]
|
|
160
159
|
desc_se = ', '.join(desc_se)
|
|
161
160
|
return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
|
|
162
|
-
|
|
161
|
+
|
|
163
162
|
def __str__(self):
|
|
164
163
|
"""Prediction statistics with standard error in parenthesis"""
|
|
165
164
|
if not isinstance(self.statistic, dict):
|
|
@@ -248,7 +247,7 @@ class Perf(object):
|
|
|
248
247
|
else:
|
|
249
248
|
self._best = np.array([key] * value.shape[1])
|
|
250
249
|
return self._best
|
|
251
|
-
BiB =
|
|
250
|
+
BiB = bool(self.statistic_samples.BiB)
|
|
252
251
|
keys = np.array(list(self.statistic.keys()))
|
|
253
252
|
data = np.asanyarray([self.statistic[k]
|
|
254
253
|
for k in keys])
|
|
@@ -338,6 +337,12 @@ class Perf(object):
|
|
|
338
337
|
CI:float=0.05,
|
|
339
338
|
kind:str='point', linestyle:str='none',
|
|
340
339
|
col_wrap:int=3, capsize:float=0.2,
|
|
340
|
+
comparison:bool=True,
|
|
341
|
+
right:bool=True,
|
|
342
|
+
comp_legend:str='Comparison',
|
|
343
|
+
winner_legend:str='Best',
|
|
344
|
+
tie_legend:str='Equivalent',
|
|
345
|
+
loser_legend:str='Different',
|
|
341
346
|
**kwargs):
|
|
342
347
|
"""plot with seaborn
|
|
343
348
|
|
|
@@ -358,36 +363,84 @@ class Perf(object):
|
|
|
358
363
|
>>> perf.plot()
|
|
359
364
|
"""
|
|
360
365
|
import seaborn as sns
|
|
361
|
-
if
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
366
|
+
if value_name is None:
|
|
367
|
+
if self.score_func is not None:
|
|
368
|
+
value_name = 'Score'
|
|
369
|
+
else:
|
|
370
|
+
value_name = 'Error'
|
|
371
|
+
if not isinstance(self.statistic, dict):
|
|
372
|
+
comparison = False
|
|
373
|
+
best = self.best
|
|
374
|
+
if isinstance(best, np.ndarray):
|
|
375
|
+
if best.shape[0] < col_wrap:
|
|
376
|
+
col_wrap = best.shape[0]
|
|
365
377
|
df = self.dataframe(value_name=value_name, var_name=var_name,
|
|
366
|
-
alg_legend=alg_legend, perf_names=perf_names
|
|
378
|
+
alg_legend=alg_legend, perf_names=perf_names,
|
|
379
|
+
comparison=comparison, alpha=CI, right=right,
|
|
380
|
+
comp_legend=comp_legend,
|
|
381
|
+
winner_legend=winner_legend,
|
|
382
|
+
tie_legend=tie_legend,
|
|
383
|
+
loser_legend=loser_legend)
|
|
367
384
|
if var_name not in df.columns:
|
|
368
385
|
var_name = None
|
|
369
386
|
col_wrap = None
|
|
370
387
|
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
388
|
+
if comparison:
|
|
389
|
+
kwargs.update(dict(hue=comp_legend))
|
|
371
390
|
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
372
391
|
y=alg_legend, col=var_name,
|
|
373
392
|
kind=kind, linestyle=linestyle,
|
|
374
393
|
col_wrap=col_wrap, capsize=capsize, **kwargs)
|
|
375
394
|
return f_grid
|
|
376
395
|
|
|
377
|
-
|
|
378
|
-
|
|
396
|
+
def dataframe(self, comparison:bool=False,
|
|
397
|
+
right:bool=True,
|
|
398
|
+
alpha:float=0.05,
|
|
399
|
+
value_name:str='Score',
|
|
379
400
|
var_name:str='Performance',
|
|
380
401
|
alg_legend:str='Algorithm',
|
|
402
|
+
comp_legend:str='Comparison',
|
|
403
|
+
winner_legend:str='Best',
|
|
404
|
+
tie_legend:str='Equivalent',
|
|
405
|
+
loser_legend:str='Different',
|
|
381
406
|
perf_names:str=None):
|
|
382
407
|
"""Dataframe"""
|
|
383
408
|
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
384
409
|
func_name = self.statistic_func.__name__
|
|
385
410
|
perf_names = [f'{func_name}({i})'
|
|
386
411
|
for i, k in enumerate(self.best)]
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
412
|
+
df = dataframe(self, value_name=value_name,
|
|
413
|
+
var_name=var_name,
|
|
414
|
+
alg_legend=alg_legend,
|
|
415
|
+
perf_names=perf_names)
|
|
416
|
+
if not comparison:
|
|
417
|
+
return df
|
|
418
|
+
df[comp_legend] = tie_legend
|
|
419
|
+
diff = self.difference()
|
|
420
|
+
best = self.best
|
|
421
|
+
if isinstance(best, str):
|
|
422
|
+
for name, p in diff.p_value(right=right).items():
|
|
423
|
+
if p >= alpha:
|
|
424
|
+
continue
|
|
425
|
+
df.loc[df[alg_legend] == name, comp_legend] = loser_legend
|
|
426
|
+
df.loc[df[alg_legend] == best, comp_legend] = winner_legend
|
|
427
|
+
else:
|
|
428
|
+
p_values = diff.p_value(right=right)
|
|
429
|
+
systems = list(p_values.keys())
|
|
430
|
+
p_values = np.array([p_values[k] for k in systems])
|
|
431
|
+
for name, p_value, winner in zip(perf_names,
|
|
432
|
+
p_values.T,
|
|
433
|
+
best):
|
|
434
|
+
mask = df[var_name] == name
|
|
435
|
+
for alg, p in zip(systems, p_value):
|
|
436
|
+
if p >= alpha and winner != alg:
|
|
437
|
+
continue
|
|
438
|
+
_ = mask & (df[alg_legend] == alg)
|
|
439
|
+
if winner == alg:
|
|
440
|
+
df.loc[_, comp_legend] = winner_legend
|
|
441
|
+
else:
|
|
442
|
+
df.loc[_, comp_legend] = loser_legend
|
|
443
|
+
return df
|
|
391
444
|
|
|
392
445
|
@property
|
|
393
446
|
def n_jobs(self):
|
|
@@ -17,12 +17,30 @@ from sklearn.base import clone
|
|
|
17
17
|
from sklearn.svm import LinearSVC
|
|
18
18
|
from sklearn.ensemble import RandomForestClassifier
|
|
19
19
|
from sklearn.naive_bayes import GaussianNB
|
|
20
|
-
from sklearn.datasets import load_iris, load_digits
|
|
20
|
+
from sklearn.datasets import load_iris, load_digits, load_breast_cancer
|
|
21
21
|
from sklearn.model_selection import train_test_split
|
|
22
22
|
import pandas as pd
|
|
23
23
|
from CompStats.tests.test_performance import DATA
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
def test_Perf_plot_col_wrap():
|
|
27
|
+
"""Test plot when 2 classes"""
|
|
28
|
+
from CompStats.metrics import f1_score
|
|
29
|
+
|
|
30
|
+
X, y = load_breast_cancer(return_X_y=True)
|
|
31
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
32
|
+
X_train, X_val, y_train, y_val = _
|
|
33
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
34
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
35
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
36
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
37
|
+
average=None,
|
|
38
|
+
num_samples=50)
|
|
39
|
+
score(nb.predict(X_val))
|
|
40
|
+
score(svm.predict(X_val))
|
|
41
|
+
score.plot()
|
|
42
|
+
|
|
43
|
+
|
|
26
44
|
def test_Difference_dataframe():
|
|
27
45
|
"""Test Difference dataframe"""
|
|
28
46
|
from CompStats.metrics import f1_score
|
|
@@ -63,15 +63,18 @@ def dataframe(instance, value_name:str='Score',
|
|
|
63
63
|
perf_names:list=None):
|
|
64
64
|
"""Dataframe"""
|
|
65
65
|
import pandas as pd
|
|
66
|
+
statistic = instance.statistic
|
|
67
|
+
if not isinstance(statistic, dict):
|
|
68
|
+
iter = instance.statistic_samples.keys()
|
|
69
|
+
else:
|
|
70
|
+
iter = statistic
|
|
66
71
|
if isinstance(instance.best, str):
|
|
67
|
-
|
|
72
|
+
calls = instance.statistic_samples.calls
|
|
73
|
+
df = pd.DataFrame({k: calls[k]
|
|
74
|
+
for k in iter if k in calls})
|
|
68
75
|
return df.melt(var_name=alg_legend,
|
|
69
76
|
value_name=value_name)
|
|
70
77
|
df = pd.DataFrame()
|
|
71
|
-
if not isinstance(instance.statistic, dict):
|
|
72
|
-
iter = instance.statistic_samples.keys()
|
|
73
|
-
else:
|
|
74
|
-
iter = instance.statistic
|
|
75
78
|
for key in iter:
|
|
76
79
|
data = instance.statistic_samples[key]
|
|
77
80
|
_df = pd.DataFrame(data,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.11
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -51,7 +51,7 @@ CompStats
|
|
|
51
51
|
|
|
52
52
|
Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
|
|
53
53
|
|
|
54
|
-
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits),
|
|
54
|
+
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
|
|
55
55
|
|
|
56
56
|
>>> from sklearn.svm import LinearSVC
|
|
57
57
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
@@ -75,10 +75,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
75
75
|
>>> score
|
|
76
76
|
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
77
77
|
|
|
78
|
-
The previous code shows the macro-f1 score and
|
|
78
|
+
The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
|
|
79
79
|
|
|
80
|
-
>>> score.statistic
|
|
81
|
-
0.
|
|
80
|
+
>>> score.statistic, score.se
|
|
81
|
+
(0.9521479775366307, 0.009717884979482313)
|
|
82
82
|
|
|
83
83
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
84
84
|
|
|
@@ -87,28 +87,36 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
87
87
|
<Perf(score_func=f1_score)>
|
|
88
88
|
Statistic with its standard error (se)
|
|
89
89
|
statistic (se)
|
|
90
|
-
0.
|
|
91
|
-
0.
|
|
90
|
+
0.9720 (0.0076) <= Random Forest
|
|
91
|
+
0.9521 (0.0097) <= alg-1
|
|
92
92
|
|
|
93
|
-
Let us incorporate another
|
|
93
|
+
Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
|
|
94
94
|
|
|
95
95
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
96
96
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
97
97
|
<Perf(score_func=f1_score)>
|
|
98
98
|
Statistic with its standard error (se)
|
|
99
99
|
statistic (se)
|
|
100
|
-
0.
|
|
101
|
-
0.
|
|
102
|
-
0.
|
|
100
|
+
0.9759 (0.0068) <= Hist. Grad. Boost. Tree
|
|
101
|
+
0.9720 (0.0076) <= Random Forest
|
|
102
|
+
0.9521 (0.0097) <= alg-1
|
|
103
|
+
0.8266 (0.0159) <= Naive Bayes
|
|
103
104
|
|
|
104
|
-
The
|
|
105
|
+
The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
|
|
106
|
+
|
|
107
|
+
>>> score.plot()
|
|
108
|
+
|
|
109
|
+
.. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
|
|
110
|
+
|
|
111
|
+
The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
|
|
105
112
|
|
|
106
113
|
>>> diff = score.difference()
|
|
107
114
|
>>> diff
|
|
108
115
|
<Difference>
|
|
109
|
-
difference p-values w.r.t
|
|
116
|
+
difference p-values w.r.t Hist. Grad. Boost. Tree
|
|
110
117
|
0.0000 <= Naive Bayes
|
|
111
|
-
0.
|
|
118
|
+
0.0100 <= alg-1
|
|
119
|
+
0.3240 <= Random Forest
|
|
112
120
|
|
|
113
121
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
114
122
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.11
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -51,7 +51,7 @@ CompStats
|
|
|
51
51
|
|
|
52
52
|
Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
|
|
53
53
|
|
|
54
|
-
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits),
|
|
54
|
+
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
|
|
55
55
|
|
|
56
56
|
>>> from sklearn.svm import LinearSVC
|
|
57
57
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
@@ -75,10 +75,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
75
75
|
>>> score
|
|
76
76
|
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
77
77
|
|
|
78
|
-
The previous code shows the macro-f1 score and
|
|
78
|
+
The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
|
|
79
79
|
|
|
80
|
-
>>> score.statistic
|
|
81
|
-
0.
|
|
80
|
+
>>> score.statistic, score.se
|
|
81
|
+
(0.9521479775366307, 0.009717884979482313)
|
|
82
82
|
|
|
83
83
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
84
84
|
|
|
@@ -87,28 +87,36 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
87
87
|
<Perf(score_func=f1_score)>
|
|
88
88
|
Statistic with its standard error (se)
|
|
89
89
|
statistic (se)
|
|
90
|
-
0.
|
|
91
|
-
0.
|
|
90
|
+
0.9720 (0.0076) <= Random Forest
|
|
91
|
+
0.9521 (0.0097) <= alg-1
|
|
92
92
|
|
|
93
|
-
Let us incorporate another
|
|
93
|
+
Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
|
|
94
94
|
|
|
95
95
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
96
96
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
97
97
|
<Perf(score_func=f1_score)>
|
|
98
98
|
Statistic with its standard error (se)
|
|
99
99
|
statistic (se)
|
|
100
|
-
0.
|
|
101
|
-
0.
|
|
102
|
-
0.
|
|
100
|
+
0.9759 (0.0068) <= Hist. Grad. Boost. Tree
|
|
101
|
+
0.9720 (0.0076) <= Random Forest
|
|
102
|
+
0.9521 (0.0097) <= alg-1
|
|
103
|
+
0.8266 (0.0159) <= Naive Bayes
|
|
103
104
|
|
|
104
|
-
The
|
|
105
|
+
The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
|
|
106
|
+
|
|
107
|
+
>>> score.plot()
|
|
108
|
+
|
|
109
|
+
.. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
|
|
110
|
+
|
|
111
|
+
The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
|
|
105
112
|
|
|
106
113
|
>>> diff = score.difference()
|
|
107
114
|
>>> diff
|
|
108
115
|
<Difference>
|
|
109
|
-
difference p-values w.r.t
|
|
116
|
+
difference p-values w.r.t Hist. Grad. Boost. Tree
|
|
110
117
|
0.0000 <= Naive Bayes
|
|
111
|
-
0.
|
|
118
|
+
0.0100 <= alg-1
|
|
119
|
+
0.3240 <= Random Forest
|
|
112
120
|
|
|
113
121
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
114
122
|
|
|
@@ -27,7 +27,7 @@ CompStats
|
|
|
27
27
|
|
|
28
28
|
Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
|
|
29
29
|
|
|
30
|
-
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits),
|
|
30
|
+
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
|
|
31
31
|
|
|
32
32
|
>>> from sklearn.svm import LinearSVC
|
|
33
33
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
@@ -51,10 +51,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
51
51
|
>>> score
|
|
52
52
|
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
53
53
|
|
|
54
|
-
The previous code shows the macro-f1 score and
|
|
54
|
+
The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
|
|
55
55
|
|
|
56
|
-
>>> score.statistic
|
|
57
|
-
0.
|
|
56
|
+
>>> score.statistic, score.se
|
|
57
|
+
(0.9521479775366307, 0.009717884979482313)
|
|
58
58
|
|
|
59
59
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
60
60
|
|
|
@@ -63,28 +63,36 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
63
63
|
<Perf(score_func=f1_score)>
|
|
64
64
|
Statistic with its standard error (se)
|
|
65
65
|
statistic (se)
|
|
66
|
-
0.
|
|
67
|
-
0.
|
|
66
|
+
0.9720 (0.0076) <= Random Forest
|
|
67
|
+
0.9521 (0.0097) <= alg-1
|
|
68
68
|
|
|
69
|
-
Let us incorporate another
|
|
69
|
+
Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
|
|
70
70
|
|
|
71
71
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
72
72
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
73
73
|
<Perf(score_func=f1_score)>
|
|
74
74
|
Statistic with its standard error (se)
|
|
75
75
|
statistic (se)
|
|
76
|
-
0.
|
|
77
|
-
0.
|
|
78
|
-
0.
|
|
76
|
+
0.9759 (0.0068) <= Hist. Grad. Boost. Tree
|
|
77
|
+
0.9720 (0.0076) <= Random Forest
|
|
78
|
+
0.9521 (0.0097) <= alg-1
|
|
79
|
+
0.8266 (0.0159) <= Naive Bayes
|
|
79
80
|
|
|
80
|
-
The
|
|
81
|
+
The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
|
|
82
|
+
|
|
83
|
+
>>> score.plot()
|
|
84
|
+
|
|
85
|
+
.. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
|
|
86
|
+
|
|
87
|
+
The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
|
|
81
88
|
|
|
82
89
|
>>> diff = score.difference()
|
|
83
90
|
>>> diff
|
|
84
91
|
<Difference>
|
|
85
|
-
difference p-values w.r.t
|
|
92
|
+
difference p-values w.r.t Hist. Grad. Boost. Tree
|
|
86
93
|
0.0000 <= Naive Bayes
|
|
87
|
-
0.
|
|
94
|
+
0.0100 <= alg-1
|
|
95
|
+
0.3240 <= Random Forest
|
|
88
96
|
|
|
89
97
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
90
98
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|