CompStats 0.1.10__tar.gz → 0.1.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {compstats-0.1.10 → compstats-0.1.12}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.10 → compstats-0.1.12}/CompStats/interface.py +85 -17
  3. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_interface.py +26 -1
  4. {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/PKG-INFO +27 -15
  5. {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/requires.txt +1 -0
  6. {compstats-0.1.10 → compstats-0.1.12}/PKG-INFO +27 -15
  7. {compstats-0.1.10 → compstats-0.1.12}/README.rst +23 -13
  8. {compstats-0.1.10 → compstats-0.1.12}/pyproject.toml +5 -1
  9. {compstats-0.1.10 → compstats-0.1.12}/CompStats/bootstrap.py +0 -0
  10. {compstats-0.1.10 → compstats-0.1.12}/CompStats/measurements.py +0 -0
  11. {compstats-0.1.10 → compstats-0.1.12}/CompStats/metrics.py +0 -0
  12. {compstats-0.1.10 → compstats-0.1.12}/CompStats/performance.py +0 -0
  13. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/__init__.py +0 -0
  14. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/data.csv +0 -0
  15. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_bootstrap.py +0 -0
  16. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_measurements.py +0 -0
  17. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_metrics.py +0 -0
  18. {compstats-0.1.10 → compstats-0.1.12}/CompStats/tests/test_performance.py +0 -0
  19. {compstats-0.1.10 → compstats-0.1.12}/CompStats/utils.py +0 -0
  20. {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/SOURCES.txt +0 -0
  21. {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/dependency_links.txt +0 -0
  22. {compstats-0.1.10 → compstats-0.1.12}/CompStats.egg-info/top_level.txt +0 -0
  23. {compstats-0.1.10 → compstats-0.1.12}/LICENSE +0 -0
  24. {compstats-0.1.10 → compstats-0.1.12}/MANIFEST.in +0 -0
  25. {compstats-0.1.10 → compstats-0.1.12}/setup.cfg +0 -0
  26. {compstats-0.1.10 → compstats-0.1.12}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.10'
14
+ __version__ = '0.1.12'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -20,7 +20,6 @@ from CompStats.bootstrap import StatisticSamples
20
20
  from CompStats.utils import progress_bar
21
21
  from CompStats import measurements
22
22
  from CompStats.measurements import SE
23
- from CompStats.performance import plot_performance, plot_difference
24
23
  from CompStats.utils import dataframe
25
24
 
26
25
 
@@ -91,6 +90,7 @@ class Perf(object):
91
90
 
92
91
  """
93
92
  def __init__(self, y_true, *y_pred,
93
+ name:str=None,
94
94
  score_func=balanced_accuracy_score,
95
95
  error_func=None,
96
96
  num_samples: int=500,
@@ -101,8 +101,13 @@ class Perf(object):
101
101
  self.score_func = score_func
102
102
  self.error_func = error_func
103
103
  algs = {}
104
- for k, v in enumerate(y_pred):
105
- algs[f'alg-{k+1}'] = np.asanyarray(v)
104
+ if name is not None:
105
+ if isinstance(name, str):
106
+ name = [name]
107
+ else:
108
+ name = [f'alg-{k+1}' for k, _ in enumerate(y_pred)]
109
+ for key, v in zip(name, y_pred):
110
+ algs[key] = np.asanyarray(v)
106
111
  algs.update(**kwargs)
107
112
  self.predictions = algs
108
113
  self.y_true = y_true
@@ -187,6 +192,7 @@ class Perf(object):
187
192
  k = 1
188
193
  name = f'alg-{k}'
189
194
  self.best = None
195
+ self.statistic = None
190
196
  self.predictions[name] = np.asanyarray(y_pred)
191
197
  samples = self._statistic_samples
192
198
  calls = samples.calls
@@ -248,7 +254,7 @@ class Perf(object):
248
254
  else:
249
255
  self._best = np.array([key] * value.shape[1])
250
256
  return self._best
251
- BiB = True if self.statistic_samples.BiB else False
257
+ BiB = bool(self.statistic_samples.BiB)
252
258
  keys = np.array(list(self.statistic.keys()))
253
259
  data = np.asanyarray([self.statistic[k]
254
260
  for k in keys])
@@ -297,14 +303,23 @@ class Perf(object):
297
303
  >>> perf.statistic
298
304
  {'alg-1': 1.0, 'forest': 0.9500891265597148}
299
305
  """
300
-
306
+ if hasattr(self, '_statistic') and self._statistic is not None:
307
+ return self._statistic
308
+ BiB = True if self.score_func is not None else False
301
309
  data = sorted([(k, self.statistic_func(self.y_true, v))
302
310
  for k, v in self.predictions.items()],
303
- key=lambda x: self.sorting_func(x[1]),
304
- reverse=self.statistic_samples.BiB)
311
+ key=lambda x: self.sorting_func(x[1]),
312
+ reverse=BiB)
305
313
  if len(data) == 1:
306
- return data[0][1]
307
- return dict(data)
314
+ self._statistic = data[0][1]
315
+ else:
316
+ self._statistic = dict(data)
317
+ return self._statistic
318
+
319
+ @statistic.setter
320
+ def statistic(self, value):
321
+ """statistic setter"""
322
+ self._statistic = value
308
323
 
309
324
  @property
310
325
  def se(self):
@@ -338,6 +353,12 @@ class Perf(object):
338
353
  CI:float=0.05,
339
354
  kind:str='point', linestyle:str='none',
340
355
  col_wrap:int=3, capsize:float=0.2,
356
+ comparison:bool=True,
357
+ right:bool=True,
358
+ comp_legend:str='Comparison',
359
+ winner_legend:str='Best',
360
+ tie_legend:str='Equivalent',
361
+ loser_legend:str='Different',
341
362
  **kwargs):
342
363
  """plot with seaborn
343
364
 
@@ -363,32 +384,79 @@ class Perf(object):
363
384
  value_name = 'Score'
364
385
  else:
365
386
  value_name = 'Error'
387
+ if not isinstance(self.statistic, dict):
388
+ comparison = False
389
+ best = self.best
390
+ if isinstance(best, np.ndarray):
391
+ if best.shape[0] < col_wrap:
392
+ col_wrap = best.shape[0]
366
393
  df = self.dataframe(value_name=value_name, var_name=var_name,
367
- alg_legend=alg_legend, perf_names=perf_names)
394
+ alg_legend=alg_legend, perf_names=perf_names,
395
+ comparison=comparison, alpha=CI, right=right,
396
+ comp_legend=comp_legend,
397
+ winner_legend=winner_legend,
398
+ tie_legend=tie_legend,
399
+ loser_legend=loser_legend)
368
400
  if var_name not in df.columns:
369
401
  var_name = None
370
402
  col_wrap = None
371
403
  ci = lambda x: measurements.CI(x, alpha=CI)
404
+ if comparison:
405
+ kwargs.update(dict(hue=comp_legend))
372
406
  f_grid = sns.catplot(df, x=value_name, errorbar=ci,
373
407
  y=alg_legend, col=var_name,
374
408
  kind=kind, linestyle=linestyle,
375
409
  col_wrap=col_wrap, capsize=capsize, **kwargs)
376
410
  return f_grid
377
411
 
378
-
379
- def dataframe(self, value_name:str='Score',
412
+ def dataframe(self, comparison:bool=False,
413
+ right:bool=True,
414
+ alpha:float=0.05,
415
+ value_name:str='Score',
380
416
  var_name:str='Performance',
381
417
  alg_legend:str='Algorithm',
418
+ comp_legend:str='Comparison',
419
+ winner_legend:str='Best',
420
+ tie_legend:str='Equivalent',
421
+ loser_legend:str='Different',
382
422
  perf_names:str=None):
383
423
  """Dataframe"""
384
424
  if perf_names is None and isinstance(self.best, np.ndarray):
385
425
  func_name = self.statistic_func.__name__
386
426
  perf_names = [f'{func_name}({i})'
387
427
  for i, k in enumerate(self.best)]
388
- return dataframe(self, value_name=value_name,
389
- var_name=var_name,
390
- alg_legend=alg_legend,
391
- perf_names=perf_names)
428
+ df = dataframe(self, value_name=value_name,
429
+ var_name=var_name,
430
+ alg_legend=alg_legend,
431
+ perf_names=perf_names)
432
+ if not comparison:
433
+ return df
434
+ df[comp_legend] = tie_legend
435
+ diff = self.difference()
436
+ best = self.best
437
+ if isinstance(best, str):
438
+ for name, p in diff.p_value(right=right).items():
439
+ if p >= alpha:
440
+ continue
441
+ df.loc[df[alg_legend] == name, comp_legend] = loser_legend
442
+ df.loc[df[alg_legend] == best, comp_legend] = winner_legend
443
+ else:
444
+ p_values = diff.p_value(right=right)
445
+ systems = list(p_values.keys())
446
+ p_values = np.array([p_values[k] for k in systems])
447
+ for name, p_value, winner in zip(perf_names,
448
+ p_values.T,
449
+ best):
450
+ mask = df[var_name] == name
451
+ for alg, p in zip(systems, p_value):
452
+ if p >= alpha and winner != alg:
453
+ continue
454
+ _ = mask & (df[alg_legend] == alg)
455
+ if winner == alg:
456
+ df.loc[_, comp_legend] = winner_legend
457
+ else:
458
+ df.loc[_, comp_legend] = loser_legend
459
+ return df
392
460
 
393
461
  @property
394
462
  def n_jobs(self):
@@ -457,7 +525,7 @@ class Perf(object):
457
525
  algs[c] = value[c].to_numpy()
458
526
  self.predictions.update(algs)
459
527
  return
460
- self._y_true = value
528
+ self._y_true = np.asanyarray(value)
461
529
 
462
530
  @property
463
531
  def score_func(self):
@@ -17,12 +17,37 @@ from sklearn.base import clone
17
17
  from sklearn.svm import LinearSVC
18
18
  from sklearn.ensemble import RandomForestClassifier
19
19
  from sklearn.naive_bayes import GaussianNB
20
- from sklearn.datasets import load_iris, load_digits
20
+ from sklearn.datasets import load_iris, load_digits, load_breast_cancer
21
21
  from sklearn.model_selection import train_test_split
22
22
  import pandas as pd
23
23
  from CompStats.tests.test_performance import DATA
24
24
 
25
25
 
26
+ def test_Perf_name():
27
+ """Test Perf name keyword"""
28
+ from CompStats.metrics import f1_score
29
+ score = f1_score([1, 0, 1], [1, 0, 0], name='algo')
30
+ assert 'algo' in score.predictions
31
+
32
+
33
+ def test_Perf_plot_col_wrap():
34
+ """Test plot when 2 classes"""
35
+ from CompStats.metrics import f1_score
36
+
37
+ X, y = load_breast_cancer(return_X_y=True)
38
+ _ = train_test_split(X, y, test_size=0.3)
39
+ X_train, X_val, y_train, y_val = _
40
+ ens = RandomForestClassifier().fit(X_train, y_train)
41
+ nb = GaussianNB().fit(X_train, y_train)
42
+ svm = LinearSVC().fit(X_train, y_train)
43
+ score = f1_score(y_val, ens.predict(X_val),
44
+ average=None,
45
+ num_samples=50)
46
+ score(nb.predict(X_val))
47
+ score(svm.predict(X_val))
48
+ score.plot()
49
+
50
+
26
51
  def test_Difference_dataframe():
27
52
  """Test Difference dataframe"""
28
53
  from CompStats.metrics import f1_score
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: CompStats
3
- Version: 0.1.10
3
+ Version: 0.1.12
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -21,6 +21,8 @@ Requires-Dist: numpy
21
21
  Requires-Dist: scikit-learn>=1.3.0
22
22
  Requires-Dist: pandas
23
23
  Requires-Dist: seaborn>=0.13.0
24
+ Requires-Dist: statsmodels
25
+ Dynamic: license-file
24
26
 
25
27
  ====================================
26
28
  CompStats
@@ -51,7 +53,7 @@ CompStats
51
53
 
52
54
  Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
53
55
 
54
- To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), three different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
56
+ To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
55
57
 
56
58
  >>> from sklearn.svm import LinearSVC
57
59
  >>> from sklearn.naive_bayes import GaussianNB
@@ -75,10 +77,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
75
77
  >>> score
76
78
  <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
77
79
 
78
- The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
80
+ The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
79
81
 
80
- >>> score.statistic
81
- 0.9434834454375508
82
+ >>> score.statistic, score.se
83
+ (0.9521479775366307, 0.009717884979482313)
82
84
 
83
85
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
84
86
 
@@ -87,28 +89,38 @@ Continuing with the example, let us assume that one wants to test another classi
87
89
  <Perf(score_func=f1_score)>
88
90
  Statistic with its standard error (se)
89
91
  statistic (se)
90
- 0.9655 (0.0077) <= Random Forest
91
- 0.9435 (0.0099) <= alg-1
92
+ 0.9720 (0.0076) <= Random Forest
93
+ 0.9521 (0.0097) <= alg-1
92
94
 
93
- Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
95
+ Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
94
96
 
95
97
  >>> nb = GaussianNB().fit(X_train, y_train)
96
98
  >>> score(nb.predict(X_val), name='Naive Bayes')
99
+ >>> hist = HistGradientBoostingClassifier().fit(X_train, y_train)
100
+ >>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
97
101
  <Perf(score_func=f1_score)>
98
102
  Statistic with its standard error (se)
99
103
  statistic (se)
100
- 0.9655 (0.0077) <= Random Forest
101
- 0.9435 (0.0099) <= alg-1
102
- 0.8549 (0.0153) <= Naive Bayes
104
+ 0.9759 (0.0068) <= Hist. Grad. Boost. Tree
105
+ 0.9720 (0.0076) <= Random Forest
106
+ 0.9521 (0.0097) <= alg-1
107
+ 0.8266 (0.0159) <= Naive Bayes
103
108
 
104
- The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
109
+ The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
110
+
111
+ >>> score.plot()
112
+
113
+ .. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
114
+
115
+ The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
105
116
 
106
117
  >>> diff = score.difference()
107
118
  >>> diff
108
119
  <Difference>
109
- difference p-values w.r.t Random Forest
120
+ difference p-values w.r.t Hist. Grad. Boost. Tree
110
121
  0.0000 <= Naive Bayes
111
- 0.0120 <= alg-1
122
+ 0.0100 <= alg-1
123
+ 0.3240 <= Random Forest
112
124
 
113
125
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
114
126
 
@@ -2,3 +2,4 @@ numpy
2
2
  scikit-learn>=1.3.0
3
3
  pandas
4
4
  seaborn>=0.13.0
5
+ statsmodels
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.2
1
+ Metadata-Version: 2.4
2
2
  Name: CompStats
3
- Version: 0.1.10
3
+ Version: 0.1.12
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -21,6 +21,8 @@ Requires-Dist: numpy
21
21
  Requires-Dist: scikit-learn>=1.3.0
22
22
  Requires-Dist: pandas
23
23
  Requires-Dist: seaborn>=0.13.0
24
+ Requires-Dist: statsmodels
25
+ Dynamic: license-file
24
26
 
25
27
  ====================================
26
28
  CompStats
@@ -51,7 +53,7 @@ CompStats
51
53
 
52
54
  Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
53
55
 
54
- To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), three different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
56
+ To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
55
57
 
56
58
  >>> from sklearn.svm import LinearSVC
57
59
  >>> from sklearn.naive_bayes import GaussianNB
@@ -75,10 +77,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
75
77
  >>> score
76
78
  <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
77
79
 
78
- The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
80
+ The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
79
81
 
80
- >>> score.statistic
81
- 0.9434834454375508
82
+ >>> score.statistic, score.se
83
+ (0.9521479775366307, 0.009717884979482313)
82
84
 
83
85
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
84
86
 
@@ -87,28 +89,38 @@ Continuing with the example, let us assume that one wants to test another classi
87
89
  <Perf(score_func=f1_score)>
88
90
  Statistic with its standard error (se)
89
91
  statistic (se)
90
- 0.9655 (0.0077) <= Random Forest
91
- 0.9435 (0.0099) <= alg-1
92
+ 0.9720 (0.0076) <= Random Forest
93
+ 0.9521 (0.0097) <= alg-1
92
94
 
93
- Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
95
+ Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
94
96
 
95
97
  >>> nb = GaussianNB().fit(X_train, y_train)
96
98
  >>> score(nb.predict(X_val), name='Naive Bayes')
99
+ >>> hist = HistGradientBoostingClassifier().fit(X_train, y_train)
100
+ >>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
97
101
  <Perf(score_func=f1_score)>
98
102
  Statistic with its standard error (se)
99
103
  statistic (se)
100
- 0.9655 (0.0077) <= Random Forest
101
- 0.9435 (0.0099) <= alg-1
102
- 0.8549 (0.0153) <= Naive Bayes
104
+ 0.9759 (0.0068) <= Hist. Grad. Boost. Tree
105
+ 0.9720 (0.0076) <= Random Forest
106
+ 0.9521 (0.0097) <= alg-1
107
+ 0.8266 (0.0159) <= Naive Bayes
103
108
 
104
- The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
109
+ The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
110
+
111
+ >>> score.plot()
112
+
113
+ .. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
114
+
115
+ The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
105
116
 
106
117
  >>> diff = score.difference()
107
118
  >>> diff
108
119
  <Difference>
109
- difference p-values w.r.t Random Forest
120
+ difference p-values w.r.t Hist. Grad. Boost. Tree
110
121
  0.0000 <= Naive Bayes
111
- 0.0120 <= alg-1
122
+ 0.0100 <= alg-1
123
+ 0.3240 <= Random Forest
112
124
 
113
125
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
114
126
 
@@ -27,7 +27,7 @@ CompStats
27
27
 
28
28
  Collaborative competitions have gained popularity in the scientific and technological fields. These competitions involve defining tasks, selecting evaluation scores, and devising result verification methods. In the standard scenario, participants receive a training set and are expected to provide a solution for a held-out dataset kept by organizers. An essential challenge for organizers arises when comparing algorithms' performance, assessing multiple participants, and ranking them. Statistical tools are often used for this purpose; however, traditional statistical methods often fail to capture decisive differences between systems' performance. CompStats implements an evaluation methodology for statistically analyzing competition results and competition. CompStats offers several advantages, including off-the-shell comparisons with correction mechanisms and the inclusion of confidence intervals.
29
29
 
30
- To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), three different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
30
+ To illustrate the use of `CompStats`, the following snippets show an example. The instructions load the necessary libraries, including the one to obtain the problem (e.g., digits), four different classifiers, and the last line is the score used to measure the performance and compare the algorithm.
31
31
 
32
32
  >>> from sklearn.svm import LinearSVC
33
33
  >>> from sklearn.naive_bayes import GaussianNB
@@ -51,10 +51,10 @@ Once the predictions are available, it is time to measure the algorithm's perfor
51
51
  >>> score
52
52
  <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
53
53
 
54
- The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
54
+ The previous code shows the macro-f1 score and its standard error. The actual performance value is stored in the attributes `statistic` function, and `se`
55
55
 
56
- >>> score.statistic
57
- 0.9434834454375508
56
+ >>> score.statistic, score.se
57
+ (0.9521479775366307, 0.009717884979482313)
58
58
 
59
59
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
60
60
 
@@ -63,28 +63,38 @@ Continuing with the example, let us assume that one wants to test another classi
63
63
  <Perf(score_func=f1_score)>
64
64
  Statistic with its standard error (se)
65
65
  statistic (se)
66
- 0.9655 (0.0077) <= Random Forest
67
- 0.9435 (0.0099) <= alg-1
66
+ 0.9720 (0.0076) <= Random Forest
67
+ 0.9521 (0.0097) <= alg-1
68
68
 
69
- Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
69
+ Let us incorporate another predictions, now with Naive Bayes classifier, and Histogram Gradient Boosting as seen below.
70
70
 
71
71
  >>> nb = GaussianNB().fit(X_train, y_train)
72
72
  >>> score(nb.predict(X_val), name='Naive Bayes')
73
+ >>> hist = HistGradientBoostingClassifier().fit(X_train, y_train)
74
+ >>> score(hist.predict(X_val), name='Hist. Grad. Boost. Tree')
73
75
  <Perf(score_func=f1_score)>
74
76
  Statistic with its standard error (se)
75
77
  statistic (se)
76
- 0.9655 (0.0077) <= Random Forest
77
- 0.9435 (0.0099) <= alg-1
78
- 0.8549 (0.0153) <= Naive Bayes
78
+ 0.9759 (0.0068) <= Hist. Grad. Boost. Tree
79
+ 0.9720 (0.0076) <= Random Forest
80
+ 0.9521 (0.0097) <= alg-1
81
+ 0.8266 (0.0159) <= Naive Bayes
79
82
 
80
- The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
83
+ The performance, its confidence interval (5%), and a statistical comparison (5%) between the best performing system with the rest of the algorithms is depicted in the following figure.
84
+
85
+ >>> score.plot()
86
+
87
+ .. image:: https://github.com/INGEOTEC/CompStats/raw/docs/docs/source/digits_perf.png
88
+
89
+ The final step is to compare the performance of the four classifiers, which can be done with the `difference` method, as seen next.
81
90
 
82
91
  >>> diff = score.difference()
83
92
  >>> diff
84
93
  <Difference>
85
- difference p-values w.r.t Random Forest
94
+ difference p-values w.r.t Hist. Grad. Boost. Tree
86
95
  0.0000 <= Naive Bayes
87
- 0.0120 <= alg-1
96
+ 0.0100 <= alg-1
97
+ 0.3240 <= Random Forest
88
98
 
89
99
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
90
100
 
@@ -6,7 +6,8 @@ dependencies = [
6
6
  'numpy',
7
7
  'scikit-learn>=1.3.0',
8
8
  'pandas',
9
- 'seaborn>=0.13.0'
9
+ 'seaborn>=0.13.0',
10
+ 'statsmodels'
10
11
  ]
11
12
  dynamic = ['version']
12
13
 
@@ -26,6 +27,9 @@ classifiers = [
26
27
  [tool.setuptools.dynamic]
27
28
  version = {attr = 'CompStats.__version__'}
28
29
 
30
+ [tool.setuptools]
31
+ packages = ['CompStats', 'CompStats.tests']
32
+
29
33
  [project.urls]
30
34
  Homepage = "https://compstats.readthedocs.io"
31
35
  Repository = "https://github.com/INGEOTEC/CompStats"
File without changes
File without changes
File without changes
File without changes