CompStats 0.1.10__tar.gz → 0.1.12__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/__init__.py +1 -1
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/interface.py +85 -17
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_interface.py +26 -1
- {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/PKG-INFO +27 -15
- {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/requires.txt +1 -0
- {compstats-0.1.10 → compstats-0.1.12}/PKG-INFO +27 -15
- {compstats-0.1.10 → compstats-0.1.12}/README.rst +23 -13
- {compstats-0.1.10 → compstats-0.1.12}/pyproject.toml +5 -1
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/measurements.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/metrics.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/performance.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_metrics.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats/utils.py +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/LICENSE +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/MANIFEST.in +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/setup.cfg +0 -0
- {compstats-0.1.10 → compstats-0.1.12}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.12'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -20,7 +20,6 @@ from CompStats.bootstrap import StatisticSamples
|
|
|
20
20
|
from CompStats.utils import progress_bar
|
|
21
21
|
from CompStats import measurements
|
|
22
22
|
from CompStats.measurements import SE
|
|
23
|
-
from CompStats.performance import plot_performance, plot_difference
|
|
24
23
|
from CompStats.utils import dataframe
|
|
25
24
|
|
|
26
25
|
|
|
@@ -91,6 +90,7 @@ class Perf(object):
|
|
|
91
90
|
|
|
92
91
|
"""
|
|
93
92
|
def __init__(self, y_true, *y_pred,
|
|
93
|
+
name:str=None,
|
|
94
94
|
score_func=balanced_accuracy_score,
|
|
95
95
|
error_func=None,
|
|
96
96
|
num_samples: int=500,
|
|
@@ -101,8 +101,13 @@ class Perf(object):
|
|
|
101
101
|
self.score_func = score_func
|
|
102
102
|
self.error_func = error_func
|
|
103
103
|
algs = {}
|
|
104
|
-
|
|
105
|
-
|
|
104
|
+
if name is not None:
|
|
105
|
+
if isinstance(name, str):
|
|
106
|
+
name = [name]
|
|
107
|
+
else:
|
|
108
|
+
name = [f'alg-{k+1}' for k, _ in enumerate(y_pred)]
|
|
109
|
+
for key, v in zip(name, y_pred):
|
|
110
|
+
algs[key] = np.asanyarray(v)
|
|
106
111
|
algs.update(**kwargs)
|
|
107
112
|
self.predictions = algs
|
|
108
113
|
self.y_true = y_true
|
|
@@ -187,6 +192,7 @@ class Perf(object):
|
|
|
187
192
|
k = 1
|
|
188
193
|
name = f'alg-{k}'
|
|
189
194
|
self.best = None
|
|
195
|
+
self.statistic = None
|
|
190
196
|
self.predictions[name] = np.asanyarray(y_pred)
|
|
191
197
|
samples = self._statistic_samples
|
|
192
198
|
calls = samples.calls
|
|
@@ -248,7 +254,7 @@ class Perf(object):
|
|
|
248
254
|
else:
|
|
249
255
|
self._best = np.array([key] * value.shape[1])
|
|
250
256
|
return self._best
|
|
251
|
-
BiB =
|
|
257
|
+
BiB = bool(self.statistic_samples.BiB)
|
|
252
258
|
keys = np.array(list(self.statistic.keys()))
|
|
253
259
|
data = np.asanyarray([self.statistic[k]
|
|
254
260
|
for k in keys])
|
|
@@ -297,14 +303,23 @@ class Perf(object):
|
|
|
297
303
|
>>> perf.statistic
|
|
298
304
|
{'alg-1': 1.0, 'forest': 0.9500891265597148}
|
|
299
305
|
"""
|
|
300
|
-
|
|
306
|
+
if hasattr(self, '_statistic') and self._statistic is not None:
|
|
307
|
+
return self._statistic
|
|
308
|
+
BiB = True if self.score_func is not None else False
|
|
301
309
|
data = sorted([(k, self.statistic_func(self.y_true, v))
|
|
302
310
|
for k, v in self.predictions.items()],
|
|
303
|
-
key=lambda x: self.sorting_func(x[1]),
|
|
304
|
-
reverse=
|
|
311
|
+
key=lambda x: self.sorting_func(x[1]),
|
|
312
|
+
reverse=BiB)
|
|
305
313
|
if len(data) == 1:
|
|
306
|
-
|
|
307
|
-
|
|
314
|
+
self._statistic = data[0][1]
|
|
315
|
+
else:
|
|
316
|
+
self._statistic = dict(data)
|
|
317
|
+
return self._statistic
|
|
318
|
+
|
|
319
|
+
@statistic.setter
|
|
320
|
+
def statistic(self, value):
|
|
321
|
+
"""statistic setter"""
|
|
322
|
+
self._statistic = value
|
|
308
323
|
|
|
309
324
|
@property
|
|
310
325
|
def se(self):
|
|
@@ -338,6 +353,12 @@ class Perf(object):
|
|
|
338
353
|
CI:float=0.05,
|
|
339
354
|
kind:str='point', linestyle:str='none',
|
|
340
355
|
col_wrap:int=3, capsize:float=0.2,
|
|
356
|
+
comparison:bool=True,
|
|
357
|
+
right:bool=True,
|
|
358
|
+
comp_legend:str='Comparison',
|
|
359
|
+
winner_legend:str='Best',
|
|
360
|
+
tie_legend:str='Equivalent',
|
|
361
|
+
loser_legend:str='Different',
|
|
341
362
|
**kwargs):
|
|
342
363
|
"""plot with seaborn
|
|
343
364
|
|
|
@@ -363,32 +384,79 @@ class Perf(object):
|
|
|
363
384
|
value_name = 'Score'
|
|
364
385
|
else:
|
|
365
386
|
value_name = 'Error'
|
|
387
|
+
if not isinstance(self.statistic, dict):
|
|
388
|
+
comparison = False
|
|
389
|
+
best = self.best
|
|
390
|
+
if isinstance(best, np.ndarray):
|
|
391
|
+
if best.shape[0] < col_wrap:
|
|
392
|
+
col_wrap = best.shape[0]
|
|
366
393
|
df = self.dataframe(value_name=value_name, var_name=var_name,
|
|
367
|
-
alg_legend=alg_legend, perf_names=perf_names
|
|
394
|
+
alg_legend=alg_legend, perf_names=perf_names,
|
|
395
|
+
comparison=comparison, alpha=CI, right=right,
|
|
396
|
+
comp_legend=comp_legend,
|
|
397
|
+
winner_legend=winner_legend,
|
|
398
|
+
tie_legend=tie_legend,
|
|
399
|
+
loser_legend=loser_legend)
|
|
368
400
|
if var_name not in df.columns:
|
|
369
401
|
var_name = None
|
|
370
402
|
col_wrap = None
|
|
371
403
|
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
404
|
+
if comparison:
|
|
405
|
+
kwargs.update(dict(hue=comp_legend))
|
|
372
406
|
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
373
407
|
y=alg_legend, col=var_name,
|
|
374
408
|
kind=kind, linestyle=linestyle,
|
|
375
409
|
col_wrap=col_wrap, capsize=capsize, **kwargs)
|
|
376
410
|
return f_grid
|
|
377
411
|
|
|
378
|
-
|
|
379
|
-
|
|
412
|
+
def dataframe(self, comparison:bool=False,
|
|
413
|
+
right:bool=True,
|
|
414
|
+
alpha:float=0.05,
|
|
415
|
+
value_name:str='Score',
|
|
380
416
|
var_name:str='Performance',
|
|
381
417
|
alg_legend:str='Algorithm',
|
|
418
|
+
comp_legend:str='Comparison',
|
|
419
|
+
winner_legend:str='Best',
|
|
420
|
+
tie_legend:str='Equivalent',
|
|
421
|
+
loser_legend:str='Different',
|
|
382
422
|
perf_names:str=None):
|
|
383
423
|
"""Dataframe"""
|
|
384
424
|
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
385
425
|
func_name = self.statistic_func.__name__
|
|
386
426
|
perf_names = [f'{func_name}({i})'
|
|
387
427
|
for i, k in enumerate(self.best)]
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
428
|
+
df = dataframe(self, value_name=value_name,
|
|
429
|
+
var_name=var_name,
|
|
430
|
+
alg_legend=alg_legend,
|
|
431
|
+
perf_names=perf_names)
|
|
432
|
+
if not comparison:
|
|
433
|
+
return df
|
|
434
|
+
df[comp_legend] = tie_legend
|
|
435
|
+
diff = self.difference()
|
|
436
|
+
best = self.best
|
|
437
|
+
if isinstance(best, str):
|
|
438
|
+
for name, p in diff.p_value(right=right).items():
|
|
439
|
+
if p >= alpha:
|
|
440
|
+
continue
|
|
441
|
+
df.loc[df[alg_legend] == name, comp_legend] = loser_legend
|
|
442
|
+
df.loc[df[alg_legend] == best, comp_legend] = winner_legend
|
|
443
|
+
else:
|
|
444
|
+
p_values = diff.p_value(right=right)
|
|
445
|
+
systems = list(p_values.keys())
|
|
446
|
+
p_values = np.array([p_values[k] for k in systems])
|
|
447
|
+
for name, p_value, winner in zip(perf_names,
|
|
448
|
+
p_values.T,
|
|
449
|
+
best):
|
|
450
|
+
mask = df[var_name] == name
|
|
451
|
+
for alg, p in zip(systems, p_value):
|
|
452
|
+
if p >= alpha and winner != alg:
|
|
453
|
+
continue
|
|
454
|
+
_ = mask & (df[alg_legend] == alg)
|
|
455
|
+
if winner == alg:
|
|
456
|
+
df.loc[_, comp_legend] = winner_legend
|
|
457
|
+
else:
|
|
458
|
+
df.loc[_, comp_legend] = loser_legend
|
|
459
|
+
return df
|
|
392
460
|
|
|
393
461
|
@property
|
|
394
462
|
def n_jobs(self):
|
|
@@ -457,7 +525,7 @@ class Perf(object):
|
|
|
457
525
|
algs[c] = value[c].to_numpy()
|
|
458
526
|
self.predictions.update(algs)
|
|
459
527
|
return
|
|
460
|
-
self._y_true = value
|
|
528
|
+
self._y_true = np.asanyarray(value)
|
|
461
529
|
|
|
462
530
|
@property
|
|
463
531
|
def score_func(self):
|
|
@@ -17,12 +17,37 @@ from sklearn.base import clone
|
|
|
17
17
|
from sklearn.svm import LinearSVC
|
|
18
18
|
from sklearn.ensemble import RandomForestClassifier
|
|
19
19
|
from sklearn.naive_bayes import GaussianNB
|
|
20
|
-
from sklearn.datasets import load_iris, load_digits
|
|
20
|
+
from sklearn.datasets import load_iris, load_digits, load_breast_cancer
|
|
21
21
|
from sklearn.model_selection import train_test_split
|
|
22
22
|
import pandas as pd
|
|
23
23
|
from CompStats.tests.test_performance import DATA
|
|
24
24
|
|
|
25
25
|
|
|
26
|
+
def test_Perf_name():
|
|
27
|
+
"""Test Perf name keyword"""
|
|
28
|
+
from CompStats.metrics import f1_score
|
|
29
|
+
score = f1_score([1, 0, 1], [1, 0, 0], name='algo')
|
|
30
|
+
assert 'algo' in score.predictions
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_Perf_plot_col_wrap():
|
|
34
|
+
"""Test plot when 2 classes"""
|
|
35
|
+
from CompStats.metrics import f1_score
|
|
36
|
+
|
|
37
|
+
X, y = load_breast_cancer(return_X_y=True)
|
|
38
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
39
|
+
X_train, X_val, y_train, y_val = _
|
|
40
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
41
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
42
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
43
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
44
|
+
average=None,
|
|
45
|
+
num_samples=50)
|
|
46
|
+
score(nb.predict(X_val))
|
|
47
|
+
score(svm.predict(X_val))
|
|
48
|
+
score.plot()
|
|
49
|
+
|
|
50
|
+
|
|
26
51
|
def test_Difference_dataframe():
|
|
27
52
|
"""Test Difference dataframe"""
|
|
28
53
|
from CompStats.metrics import f1_score
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.12
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -21,6 +21,8 @@ Requires-Dist: numpy
|
|
|
21
21
|
Requires-Dist: scikit-learn>=1.3.0
|
|
22
22
|
Requires-Dist: pandas
|
|
23
23
|
Requires-Dist: seaborn>=0.13.0
|
|
24
|
+
Requires-Dist: statsmodels
|
|
25
|
+
Dynamic: license-file
|
|
24
26
|
|
|
25
27
|
====================================
|
|
26
28
|
CompStats
|
|
@@ -51,7 +53,7 @@ CompStats
|
|
|
51
53
|
|
|
52
54
|
Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
|
|
53
55
|
|
|
54
|
-
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits),
|
|
56
|
+
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
|
|
55
57
|
|
|
56
58
|
>>> from sklearn.svm import LinearSVC
|
|
57
59
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
@@ -75,10 +77,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
75
77
|
>>> score
|
|
76
78
|
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
77
79
|
|
|
78
|
-
The previous code shows the macro-f1 score and
|
|
80
|
+
The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
|
|
79
81
|
|
|
80
|
-
>>> score.statistic
|
|
81
|
-
0.
|
|
82
|
+
>>> score.statistic, score.se
|
|
83
|
+
(0.9521479775366307, 0.009717884979482313)
|
|
82
84
|
|
|
83
85
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
84
86
|
|
|
@@ -87,28 +89,38 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
87
89
|
<Perf(score_func=f1_score)>
|
|
88
90
|
Statistic with its standard error (se)
|
|
89
91
|
statistic (se)
|
|
90
|
-
0.
|
|
91
|
-
0.
|
|
92
|
+
0.9720 (0.0076) <= Random Forest
|
|
93
|
+
0.9521 (0.0097) <= alg-1
|
|
92
94
|
|
|
93
|
-
Let us incorporate another
|
|
95
|
+
Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
|
|
94
96
|
|
|
95
97
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
96
98
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
99
|
+
>>> hist = HistGradientBoostingClassifier().fit(X_train, y_train)
|
|
100
|
+
>>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
|
|
97
101
|
<Perf(score_func=f1_score)>
|
|
98
102
|
Statistic with its standard error (se)
|
|
99
103
|
statistic (se)
|
|
100
|
-
0.
|
|
101
|
-
0.
|
|
102
|
-
0.
|
|
104
|
+
0.9759 (0.0068) <= Hist. Grad. Boost. Tree
|
|
105
|
+
0.9720 (0.0076) <= Random Forest
|
|
106
|
+
0.9521 (0.0097) <= alg-1
|
|
107
|
+
0.8266 (0.0159) <= Naive Bayes
|
|
103
108
|
|
|
104
|
-
The
|
|
109
|
+
The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
|
|
110
|
+
|
|
111
|
+
>>> score.plot()
|
|
112
|
+
|
|
113
|
+
.. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
|
|
114
|
+
|
|
115
|
+
The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
|
|
105
116
|
|
|
106
117
|
>>> diff = score.difference()
|
|
107
118
|
>>> diff
|
|
108
119
|
<Difference>
|
|
109
|
-
difference p-values w.r.t
|
|
120
|
+
difference p-values w.r.t Hist. Grad. Boost. Tree
|
|
110
121
|
0.0000 <= Naive Bayes
|
|
111
|
-
0.
|
|
122
|
+
0.0100 <= alg-1
|
|
123
|
+
0.3240 <= Random Forest
|
|
112
124
|
|
|
113
125
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
114
126
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.12
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -21,6 +21,8 @@ Requires-Dist: numpy
|
|
|
21
21
|
Requires-Dist: scikit-learn>=1.3.0
|
|
22
22
|
Requires-Dist: pandas
|
|
23
23
|
Requires-Dist: seaborn>=0.13.0
|
|
24
|
+
Requires-Dist: statsmodels
|
|
25
|
+
Dynamic: license-file
|
|
24
26
|
|
|
25
27
|
====================================
|
|
26
28
|
CompStats
|
|
@@ -51,7 +53,7 @@ CompStats
|
|
|
51
53
|
|
|
52
54
|
Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
|
|
53
55
|
|
|
54
|
-
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits),
|
|
56
|
+
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
|
|
55
57
|
|
|
56
58
|
>>> from sklearn.svm import LinearSVC
|
|
57
59
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
@@ -75,10 +77,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
75
77
|
>>> score
|
|
76
78
|
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
77
79
|
|
|
78
|
-
The previous code shows the macro-f1 score and
|
|
80
|
+
The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
|
|
79
81
|
|
|
80
|
-
>>> score.statistic
|
|
81
|
-
0.
|
|
82
|
+
>>> score.statistic, score.se
|
|
83
|
+
(0.9521479775366307, 0.009717884979482313)
|
|
82
84
|
|
|
83
85
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
84
86
|
|
|
@@ -87,28 +89,38 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
87
89
|
<Perf(score_func=f1_score)>
|
|
88
90
|
Statistic with its standard error (se)
|
|
89
91
|
statistic (se)
|
|
90
|
-
0.
|
|
91
|
-
0.
|
|
92
|
+
0.9720 (0.0076) <= Random Forest
|
|
93
|
+
0.9521 (0.0097) <= alg-1
|
|
92
94
|
|
|
93
|
-
Let us incorporate another
|
|
95
|
+
Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
|
|
94
96
|
|
|
95
97
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
96
98
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
99
|
+
>>> hist = HistGradientBoostingClassifier().fit(X_train, y_train)
|
|
100
|
+
>>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
|
|
97
101
|
<Perf(score_func=f1_score)>
|
|
98
102
|
Statistic with its standard error (se)
|
|
99
103
|
statistic (se)
|
|
100
|
-
0.
|
|
101
|
-
0.
|
|
102
|
-
0.
|
|
104
|
+
0.9759 (0.0068) <= Hist. Grad. Boost. Tree
|
|
105
|
+
0.9720 (0.0076) <= Random Forest
|
|
106
|
+
0.9521 (0.0097) <= alg-1
|
|
107
|
+
0.8266 (0.0159) <= Naive Bayes
|
|
103
108
|
|
|
104
|
-
The
|
|
109
|
+
The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
|
|
110
|
+
|
|
111
|
+
>>> score.plot()
|
|
112
|
+
|
|
113
|
+
.. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
|
|
114
|
+
|
|
115
|
+
The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
|
|
105
116
|
|
|
106
117
|
>>> diff = score.difference()
|
|
107
118
|
>>> diff
|
|
108
119
|
<Difference>
|
|
109
|
-
difference p-values w.r.t
|
|
120
|
+
difference p-values w.r.t Hist. Grad. Boost. Tree
|
|
110
121
|
0.0000 <= Naive Bayes
|
|
111
|
-
0.
|
|
122
|
+
0.0100 <= alg-1
|
|
123
|
+
0.3240 <= Random Forest
|
|
112
124
|
|
|
113
125
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
114
126
|
|
|
@@ -27,7 +27,7 @@ CompStats
|
|
|
27
27
|
|
|
28
28
|
Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
|
|
29
29
|
|
|
30
|
-
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits),
|
|
30
|
+
To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
|
|
31
31
|
|
|
32
32
|
>>> from sklearn.svm import LinearSVC
|
|
33
33
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
@@ -51,10 +51,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
|
|
|
51
51
|
>>> score
|
|
52
52
|
<Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
|
|
53
53
|
|
|
54
|
-
The previous code shows the macro-f1 score and
|
|
54
|
+
The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
|
|
55
55
|
|
|
56
|
-
>>> score.statistic
|
|
57
|
-
0.
|
|
56
|
+
>>> score.statistic, score.se
|
|
57
|
+
(0.9521479775366307, 0.009717884979482313)
|
|
58
58
|
|
|
59
59
|
Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
|
|
60
60
|
|
|
@@ -63,28 +63,38 @@ Continuing with the example, let us assume that one wants to test another classi
|
|
|
63
63
|
<Perf(score_func=f1_score)>
|
|
64
64
|
Statistic with its standard error (se)
|
|
65
65
|
statistic (se)
|
|
66
|
-
0.
|
|
67
|
-
0.
|
|
66
|
+
0.9720 (0.0076) <= Random Forest
|
|
67
|
+
0.9521 (0.0097) <= alg-1
|
|
68
68
|
|
|
69
|
-
Let us incorporate another
|
|
69
|
+
Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
|
|
70
70
|
|
|
71
71
|
>>> nb = GaussianNB().fit(X_train, y_train)
|
|
72
72
|
>>> score(nb.predict(X_val), name='Naive Bayes')
|
|
73
|
+
>>> hist = HistGradientBoostingClassifier().fit(X_train, y_train)
|
|
74
|
+
>>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
|
|
73
75
|
<Perf(score_func=f1_score)>
|
|
74
76
|
Statistic with its standard error (se)
|
|
75
77
|
statistic (se)
|
|
76
|
-
0.
|
|
77
|
-
0.
|
|
78
|
-
0.
|
|
78
|
+
0.9759 (0.0068) <= Hist. Grad. Boost. Tree
|
|
79
|
+
0.9720 (0.0076) <= Random Forest
|
|
80
|
+
0.9521 (0.0097) <= alg-1
|
|
81
|
+
0.8266 (0.0159) <= Naive Bayes
|
|
79
82
|
|
|
80
|
-
The
|
|
83
|
+
The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
|
|
84
|
+
|
|
85
|
+
>>> score.plot()
|
|
86
|
+
|
|
87
|
+
.. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
|
|
88
|
+
|
|
89
|
+
The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
|
|
81
90
|
|
|
82
91
|
>>> diff = score.difference()
|
|
83
92
|
>>> diff
|
|
84
93
|
<Difference>
|
|
85
|
-
difference p-values w.r.t
|
|
94
|
+
difference p-values w.r.t Hist. Grad. Boost. Tree
|
|
86
95
|
0.0000 <= Naive Bayes
|
|
87
|
-
0.
|
|
96
|
+
0.0100 <= alg-1
|
|
97
|
+
0.3240 <= Random Forest
|
|
88
98
|
|
|
89
99
|
The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
|
|
90
100
|
|
|
@@ -6,7 +6,8 @@ dependencies = [
|
|
|
6
6
|
'numpy',
|
|
7
7
|
'scikit-learn>=1.3.0',
|
|
8
8
|
'pandas',
|
|
9
|
-
'seaborn>=0.13.0'
|
|
9
|
+
'seaborn>=0.13.0',
|
|
10
|
+
'statsmodels'
|
|
10
11
|
]
|
|
11
12
|
dynamic = ['version']
|
|
12
13
|
|
|
@@ -26,6 +27,9 @@ classifiers = [
|
|
|
26
27
|
[tool.setuptools.dynamic]
|
|
27
28
|
version = {attr = 'CompStats.__version__'}
|
|
28
29
|
|
|
30
|
+
[tool.setuptools]
|
|
31
|
+
packages = ['CompStats', 'CompStats.tests']
|
|
32
|
+
|
|
29
33
|
[project.urls]
|
|
30
34
|
Homepage = "https://compstats.readthedocs.io"
|
|
31
35
|
Repository = "https://github.com/INGEOTEC/CompStats"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|