CompStats 0.1.9__tar.gz → 0.1.11__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {compstats-0.1.9 → compstats-0.1.11}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.9 → compstats-0.1.11}/CompStats/interface.py +67 -14
  3. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_interface.py +19 -1
  4. {compstats-0.1.9 → compstats-0.1.11}/CompStats/utils.py +8 -5
  5. {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/PKG-INFO +22 -14
  6. {compstats-0.1.9 → compstats-0.1.11}/PKG-INFO +22 -14
  7. {compstats-0.1.9 → compstats-0.1.11}/README.rst +21 -13
  8. {compstats-0.1.9 → compstats-0.1.11}/CompStats/bootstrap.py +0 -0
  9. {compstats-0.1.9 → compstats-0.1.11}/CompStats/measurements.py +0 -0
  10. {compstats-0.1.9 → compstats-0.1.11}/CompStats/metrics.py +0 -0
  11. {compstats-0.1.9 → compstats-0.1.11}/CompStats/performance.py +0 -0
  12. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/__init__.py +0 -0
  13. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/data.csv +0 -0
  14. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_bootstrap.py +0 -0
  15. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_measurements.py +0 -0
  16. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_metrics.py +0 -0
  17. {compstats-0.1.9 → compstats-0.1.11}/CompStats/tests/test_performance.py +0 -0
  18. {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/SOURCES.txt +0 -0
  19. {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/dependency_links.txt +0 -0
  20. {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/requires.txt +0 -0
  21. {compstats-0.1.9 → compstats-0.1.11}/CompStats.egg-info/top_level.txt +0 -0
  22. {compstats-0.1.9 → compstats-0.1.11}/LICENSE +0 -0
  23. {compstats-0.1.9 → compstats-0.1.11}/MANIFEST.in +0 -0
  24. {compstats-0.1.9 → compstats-0.1.11}/pyproject.toml +0 -0
  25. {compstats-0.1.9 → compstats-0.1.11}/setup.cfg +0 -0
  26. {compstats-0.1.9 → compstats-0.1.11}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.9'
14
+ __version__ = '0.1.11'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -20,7 +20,6 @@ from CompStats.bootstrap import StatisticSamples
20
20
  from CompStats.utils import progress_bar
21
21
  from CompStats import measurements
22
22
  from CompStats.measurements import SE
23
- from CompStats.performance import plot_performance, plot_difference
24
23
  from CompStats.utils import dataframe
25
24
 
26
25
 
@@ -159,7 +158,7 @@ class Perf(object):
159
158
  desc_se = [f'{k:0.4f}' for k in self.se]
160
159
  desc_se = ', '.join(desc_se)
161
160
  return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
162
-
161
+
163
162
  def __str__(self):
164
163
  """Prediction statistics with standard error in parenthesis"""
165
164
  if not isinstance(self.statistic, dict):
@@ -248,7 +247,7 @@ class Perf(object):
248
247
  else:
249
248
  self._best = np.array([key] * value.shape[1])
250
249
  return self._best
251
- BiB = True if self.statistic_samples.BiB else False
250
+ BiB = bool(self.statistic_samples.BiB)
252
251
  keys = np.array(list(self.statistic.keys()))
253
252
  data = np.asanyarray([self.statistic[k]
254
253
  for k in keys])
@@ -338,6 +337,12 @@ class Perf(object):
338
337
  CI:float=0.05,
339
338
  kind:str='point', linestyle:str='none',
340
339
  col_wrap:int=3, capsize:float=0.2,
340
+ comparison:bool=True,
341
+ right:bool=True,
342
+ comp_legend:str='Comparison',
343
+ winner_legend:str='Best',
344
+ tie_legend:str='Equivalent',
345
+ loser_legend:str='Different',
341
346
  **kwargs):
342
347
  """plot with seaborn
343
348
 
@@ -358,36 +363,84 @@ class Perf(object):
358
363
  >>> perf.plot()
359
364
  """
360
365
  import seaborn as sns
361
- if self.score_func is not None:
362
- value_name = 'Score'
363
- else:
364
- value_name = 'Error'
366
+ if value_name is None:
367
+ if self.score_func is not None:
368
+ value_name = 'Score'
369
+ else:
370
+ value_name = 'Error'
371
+ if not isinstance(self.statistic, dict):
372
+ comparison = False
373
+ best = self.best
374
+ if isinstance(best, np.ndarray):
375
+ if best.shape[0] < col_wrap:
376
+ col_wrap = best.shape[0]
365
377
  df = self.dataframe(value_name=value_name, var_name=var_name,
366
- alg_legend=alg_legend, perf_names=perf_names)
378
+ alg_legend=alg_legend, perf_names=perf_names,
379
+ comparison=comparison, alpha=CI, right=right,
380
+ comp_legend=comp_legend,
381
+ winner_legend=winner_legend,
382
+ tie_legend=tie_legend,
383
+ loser_legend=loser_legend)
367
384
  if var_name not in df.columns:
368
385
  var_name = None
369
386
  col_wrap = None
370
387
  ci = lambda x: measurements.CI(x, alpha=CI)
388
+ if comparison:
389
+ kwargs.update(dict(hue=comp_legend))
371
390
  f_grid = sns.catplot(df, x=value_name, errorbar=ci,
372
391
  y=alg_legend, col=var_name,
373
392
  kind=kind, linestyle=linestyle,
374
393
  col_wrap=col_wrap, capsize=capsize, **kwargs)
375
394
  return f_grid
376
395
 
377
-
378
- def dataframe(self, value_name:str='Score',
396
+ def dataframe(self, comparison:bool=False,
397
+ right:bool=True,
398
+ alpha:float=0.05,
399
+ value_name:str='Score',
379
400
  var_name:str='Performance',
380
401
  alg_legend:str='Algorithm',
402
+ comp_legend:str='Comparison',
403
+ winner_legend:str='Best',
404
+ tie_legend:str='Equivalent',
405
+ loser_legend:str='Different',
381
406
  perf_names:str=None):
382
407
  """Dataframe"""
383
408
  if perf_names is None and isinstance(self.best, np.ndarray):
384
409
  func_name = self.statistic_func.__name__
385
410
  perf_names = [f'{func_name}({i})'
386
411
  for i, k in enumerate(self.best)]
387
- return dataframe(self, value_name=value_name,
388
- var_name=var_name,
389
- alg_legend=alg_legend,
390
- perf_names=perf_names)
412
+ df = dataframe(self, value_name=value_name,
413
+ var_name=var_name,
414
+ alg_legend=alg_legend,
415
+ perf_names=perf_names)
416
+ if not comparison:
417
+ return df
418
+ df[comp_legend] = tie_legend
419
+ diff = self.difference()
420
+ best = self.best
421
+ if isinstance(best, str):
422
+ for name, p in diff.p_value(right=right).items():
423
+ if p >= alpha:
424
+ continue
425
+ df.loc[df[alg_legend] == name, comp_legend] = loser_legend
426
+ df.loc[df[alg_legend] == best, comp_legend] = winner_legend
427
+ else:
428
+ p_values = diff.p_value(right=right)
429
+ systems = list(p_values.keys())
430
+ p_values = np.array([p_values[k] for k in systems])
431
+ for name, p_value, winner in zip(perf_names,
432
+ p_values.T,
433
+ best):
434
+ mask = df[var_name] == name
435
+ for alg, p in zip(systems, p_value):
436
+ if p >= alpha and winner != alg:
437
+ continue
438
+ _ = mask & (df[alg_legend] == alg)
439
+ if winner == alg:
440
+ df.loc[_, comp_legend] = winner_legend
441
+ else:
442
+ df.loc[_, comp_legend] = loser_legend
443
+ return df
391
444
 
392
445
  @property
393
446
  def n_jobs(self):
@@ -17,12 +17,30 @@ from sklearn.base import clone
17
17
  from sklearn.svm import LinearSVC
18
18
  from sklearn.ensemble import RandomForestClassifier
19
19
  from sklearn.naive_bayes import GaussianNB
20
- from sklearn.datasets import load_iris, load_digits
20
+ from sklearn.datasets import load_iris, load_digits, load_breast_cancer
21
21
  from sklearn.model_selection import train_test_split
22
22
  import pandas as pd
23
23
  from CompStats.tests.test_performance import DATA
24
24
 
25
25
 
26
+ def test_Perf_plot_col_wrap():
27
+ """Test plot when 2 classes"""
28
+ from CompStats.metrics import f1_score
29
+
30
+ X, y = load_breast_cancer(return_X_y=True)
31
+ _ = train_test_split(X, y, test_size=0.3)
32
+ X_train, X_val, y_train, y_val = _
33
+ ens = RandomForestClassifier().fit(X_train, y_train)
34
+ nb = GaussianNB().fit(X_train, y_train)
35
+ svm = LinearSVC().fit(X_train, y_train)
36
+ score = f1_score(y_val, ens.predict(X_val),
37
+ average=None,
38
+ num_samples=50)
39
+ score(nb.predict(X_val))
40
+ score(svm.predict(X_val))
41
+ score.plot()
42
+
43
+
26
44
  def test_Difference_dataframe():
27
45
  """Test Difference dataframe"""
28
46
  from CompStats.metrics import f1_score
@@ -63,15 +63,18 @@ def dataframe(instance, value_name:str='Score',
63
63
  perf_names:list=None):
64
64
  """Dataframe"""
65
65
  import pandas as pd
66
+ statistic = instance.statistic
67
+ if not isinstance(statistic, dict):
68
+ iter = instance.statistic_samples.keys()
69
+ else:
70
+ iter = statistic
66
71
  if isinstance(instance.best, str):
67
- df = pd.DataFrame(dict(instance.statistic_samples.calls.items()))
72
+ calls = instance.statistic_samples.calls
73
+ df = pd.DataFrame({k: calls[k]
74
+ for k in iter if k in calls})
68
75
  return df.melt(var_name=alg_legend,
69
76
  value_name=value_name)
70
77
  df = pd.DataFrame()
71
- if not isinstance(instance.statistic, dict):
72
- iter = instance.statistic_samples.keys()
73
- else:
74
- iter = instance.statistic
75
78
  for key in iter:
76
79
  data = instance.statistic_samples[key]
77
80
  _df = pd.DataFrame(data,
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.9
3
+ Version: 0.1.11
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -51,7 +51,7 @@ CompStats
51
51
 
52
52
  Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
53
53
 
54
- To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), three different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
54
+ To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
55
55
 
56
56
  >>> from sklearn.svm import LinearSVC
57
57
  >>> from sklearn.naive_bayes import GaussianNB
@@ -75,10 +75,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
75
75
  >>> score
76
76
  <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
77
77
 
78
- The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
78
+ The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
79
79
 
80
- >>> score.statistic
81
- 0.9434834454375508
80
+ >>> score.statistic, score.se
81
+ (0.9521479775366307, 0.009717884979482313)
82
82
 
83
83
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
84
84
 
@@ -87,28 +87,36 @@ Continuing with the example, let us assume that one wants to test another classi
87
87
  <Perf(score_func=f1_score)>
88
88
  Statistic with its standard error (se)
89
89
  statistic (se)
90
- 0.9655 (0.0077) <= Random Forest
91
- 0.9435 (0.0099) <= alg-1
90
+ 0.9720 (0.0076) <= Random Forest
91
+ 0.9521 (0.0097) <= alg-1
92
92
 
93
- Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
93
+ Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
94
94
 
95
95
  >>> nb = GaussianNB().fit(X_train, y_train)
96
96
  >>> score(nb.predict(X_val), name='Naive Bayes')
97
97
  <Perf(score_func=f1_score)>
98
98
  Statistic with its standard error (se)
99
99
  statistic (se)
100
- 0.9655 (0.0077) <= Random Forest
101
- 0.9435 (0.0099) <= alg-1
102
- 0.8549 (0.0153) <= Naive Bayes
100
+ 0.9759 (0.0068) <= Hist. Grad. Boost. Tree
101
+ 0.9720 (0.0076) <= Random Forest
102
+ 0.9521 (0.0097) <= alg-1
103
+ 0.8266 (0.0159) <= Naive Bayes
103
104
 
104
- The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
105
+ The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
106
+
107
+ >>> score.plot()
108
+
109
+ .. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
110
+
111
+ The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
105
112
 
106
113
  >>> diff = score.difference()
107
114
  >>> diff
108
115
  <Difference>
109
- difference p-values w.r.t Random Forest
116
+ difference p-values w.r.t Hist. Grad. Boost. Tree
110
117
  0.0000 <= Naive Bayes
111
- 0.0120 <= alg-1
118
+ 0.0100 <= alg-1
119
+ 0.3240 <= Random Forest
112
120
 
113
121
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
114
122
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.9
3
+ Version: 0.1.11
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -51,7 +51,7 @@ CompStats
51
51
 
52
52
  Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
53
53
 
54
- To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), three different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
54
+ To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
55
55
 
56
56
  >>> from sklearn.svm import LinearSVC
57
57
  >>> from sklearn.naive_bayes import GaussianNB
@@ -75,10 +75,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
75
75
  >>> score
76
76
  <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
77
77
 
78
- The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
78
+ The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
79
79
 
80
- >>> score.statistic
81
- 0.9434834454375508
80
+ >>> score.statistic, score.se
81
+ (0.9521479775366307, 0.009717884979482313)
82
82
 
83
83
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
84
84
 
@@ -87,28 +87,36 @@ Continuing with the example, let us assume that one wants to test another classi
87
87
  <Perf(score_func=f1_score)>
88
88
  Statistic with its standard error (se)
89
89
  statistic (se)
90
- 0.9655 (0.0077) <= Random Forest
91
- 0.9435 (0.0099) <= alg-1
90
+ 0.9720 (0.0076) <= Random Forest
91
+ 0.9521 (0.0097) <= alg-1
92
92
 
93
- Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
93
+ Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
94
94
 
95
95
  >>> nb = GaussianNB().fit(X_train, y_train)
96
96
  >>> score(nb.predict(X_val), name='Naive Bayes')
97
97
  <Perf(score_func=f1_score)>
98
98
  Statistic with its standard error (se)
99
99
  statistic (se)
100
- 0.9655 (0.0077) <= Random Forest
101
- 0.9435 (0.0099) <= alg-1
102
- 0.8549 (0.0153) <= Naive Bayes
100
+ 0.9759 (0.0068) <= Hist. Grad. Boost. Tree
101
+ 0.9720 (0.0076) <= Random Forest
102
+ 0.9521 (0.0097) <= alg-1
103
+ 0.8266 (0.0159) <= Naive Bayes
103
104
 
104
- The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
105
+ The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
106
+
107
+ >>> score.plot()
108
+
109
+ .. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
110
+
111
+ The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
105
112
 
106
113
  >>> diff = score.difference()
107
114
  >>> diff
108
115
  <Difference>
109
- difference p-values w.r.t Random Forest
116
+ difference p-values w.r.t Hist. Grad. Boost. Tree
110
117
  0.0000 <= Naive Bayes
111
- 0.0120 <= alg-1
118
+ 0.0100 <= alg-1
119
+ 0.3240 <= Random Forest
112
120
 
113
121
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
114
122
 
@@ -27,7 +27,7 @@ CompStats
27
27
 
28
28
  Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
29
29
 
30
- To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), three different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
30
+ To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
31
31
 
32
32
  >>> from sklearn.svm import LinearSVC
33
33
  >>> from sklearn.naive_bayes import GaussianNB
@@ -51,10 +51,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
51
51
  >>> score
52
52
  <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
53
53
 
54
- The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
54
+ The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
55
55
 
56
- >>> score.statistic
57
- 0.9434834454375508
56
+ >>> score.statistic, score.se
57
+ (0.9521479775366307, 0.009717884979482313)
58
58
 
59
59
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
60
60
 
@@ -63,28 +63,36 @@ Continuing with the example, let us assume that one wants to test another classi
63
63
  <Perf(score_func=f1_score)>
64
64
  Statistic with its standard error (se)
65
65
  statistic (se)
66
- 0.9655 (0.0077) <= Random Forest
67
- 0.9435 (0.0099) <= alg-1
66
+ 0.9720 (0.0076) <= Random Forest
67
+ 0.9521 (0.0097) <= alg-1
68
68
 
69
- Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
69
+ Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
70
70
 
71
71
  >>> nb = GaussianNB().fit(X_train, y_train)
72
72
  >>> score(nb.predict(X_val), name='Naive Bayes')
73
73
  <Perf(score_func=f1_score)>
74
74
  Statistic with its standard error (se)
75
75
  statistic (se)
76
- 0.9655 (0.0077) <= Random Forest
77
- 0.9435 (0.0099) <= alg-1
78
- 0.8549 (0.0153) <= Naive Bayes
76
+ 0.9759 (0.0068) <= Hist. Grad. Boost. Tree
77
+ 0.9720 (0.0076) <= Random Forest
78
+ 0.9521 (0.0097) <= alg-1
79
+ 0.8266 (0.0159) <= Naive Bayes
79
80
 
80
- The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
81
+ The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
82
+
83
+ >>> score.plot()
84
+
85
+ .. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
86
+
87
+ The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
81
88
 
82
89
  >>> diff = score.difference()
83
90
  >>> diff
84
91
  <Difference>
85
- difference p-values w.r.t Random Forest
92
+ difference p-values w.r.t Hist. Grad. Boost. Tree
86
93
  0.0000 <= Naive Bayes
87
- 0.0120 <= alg-1
94
+ 0.0100 <= alg-1
95
+ 0.3240 <= Random Forest
88
96
 
89
97
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
90
98
 
File without changes
File without changes
File without changes
File without changes
File without changes