CompStats 0.1.6__tar.gz → 0.1.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {compstats-0.1.6 → compstats-0.1.7}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.6 → compstats-0.1.7}/CompStats/bootstrap.py +4 -0
  3. {compstats-0.1.6 → compstats-0.1.7}/CompStats/interface.py +84 -7
  4. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_interface.py +74 -1
  5. {compstats-0.1.6 → compstats-0.1.7}/CompStats/utils.py +26 -1
  6. {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/PKG-INFO +10 -13
  7. {compstats-0.1.6 → compstats-0.1.7}/PKG-INFO +10 -13
  8. {compstats-0.1.6 → compstats-0.1.7}/README.rst +9 -12
  9. {compstats-0.1.6 → compstats-0.1.7}/CompStats/measurements.py +0 -0
  10. {compstats-0.1.6 → compstats-0.1.7}/CompStats/metrics.py +0 -0
  11. {compstats-0.1.6 → compstats-0.1.7}/CompStats/performance.py +0 -0
  12. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/__init__.py +0 -0
  13. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/data.csv +0 -0
  14. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_bootstrap.py +0 -0
  15. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_measurements.py +0 -0
  16. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_metrics.py +0 -0
  17. {compstats-0.1.6 → compstats-0.1.7}/CompStats/tests/test_performance.py +0 -0
  18. {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/SOURCES.txt +0 -0
  19. {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/dependency_links.txt +0 -0
  20. {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/requires.txt +0 -0
  21. {compstats-0.1.6 → compstats-0.1.7}/CompStats.egg-info/top_level.txt +0 -0
  22. {compstats-0.1.6 → compstats-0.1.7}/LICENSE +0 -0
  23. {compstats-0.1.6 → compstats-0.1.7}/MANIFEST.in +0 -0
  24. {compstats-0.1.6 → compstats-0.1.7}/pyproject.toml +0 -0
  25. {compstats-0.1.6 → compstats-0.1.7}/setup.cfg +0 -0
  26. {compstats-0.1.6 → compstats-0.1.7}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.6'
14
+ __version__ = '0.1.7'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -142,6 +142,10 @@ class StatisticSamples:
142
142
  return inner(N)
143
143
  except AttributeError:
144
144
  return inner(N)
145
+
146
+ def keys(self):
147
+ """calls keys"""
148
+ return self.calls.keys()
145
149
 
146
150
  def __getitem__(self, key):
147
151
  return self.calls[key]
@@ -18,8 +18,10 @@ import pandas as pd
18
18
  import numpy as np
19
19
  from CompStats.bootstrap import StatisticSamples
20
20
  from CompStats.utils import progress_bar
21
+ from CompStats import measurements
21
22
  from CompStats.measurements import SE
22
23
  from CompStats.performance import plot_performance, plot_difference
24
+ from CompStats.utils import dataframe
23
25
 
24
26
 
25
27
  class Perf(object):
@@ -239,6 +241,13 @@ class Perf(object):
239
241
  """System with best performance"""
240
242
  if hasattr(self, '_best') and self._best is not None:
241
243
  return self._best
244
+ if not isinstance(self.statistic, dict):
245
+ key, value = list(self.statistic_samples.calls.items())[0]
246
+ if value.ndim == 1:
247
+ self._best = key
248
+ else:
249
+ self._best = np.array([key] * value.shape[1])
250
+ return self._best
242
251
  BiB = True if self.statistic_samples.BiB else False
243
252
  keys = np.array(list(self.statistic.keys()))
244
253
  data = np.asanyarray([self.statistic[k]
@@ -322,7 +331,14 @@ class Perf(object):
322
331
  return list(output.values())[0]
323
332
  return output
324
333
 
325
- def plot(self, **kwargs):
334
+ def plot(self, value_name:str=None,
335
+ var_name:str='Performance',
336
+ alg_legend:str='Algorithm',
337
+ perf_names:list=None,
338
+ CI:float=0.05,
339
+ kind:str='point', linestyle:str='none',
340
+ col_wrap:int=3, capsize:float=0.2,
341
+ **kwargs):
326
342
  """plot with seaborn
327
343
 
328
344
  >>> from sklearn.svm import LinearSVC
@@ -341,13 +357,37 @@ class Perf(object):
341
357
  forest=ens.predict(X_val))
342
358
  >>> perf.plot()
343
359
  """
360
+ import seaborn as sns
344
361
  if self.score_func is not None:
345
362
  value_name = 'Score'
346
363
  else:
347
364
  value_name = 'Error'
348
- _ = dict(value_name=value_name)
349
- _.update(kwargs)
350
- return plot_performance(self.statistic_samples, **_)
365
+ df = self.dataframe(value_name=value_name, var_name=var_name,
366
+ alg_legend=alg_legend, perf_names=perf_names)
367
+ if var_name not in df.columns:
368
+ var_name = None
369
+ col_wrap = None
370
+ ci = lambda x: measurements.CI(x, alpha=CI)
371
+ f_grid = sns.catplot(df, x=value_name, errorbar=ci,
372
+ y=alg_legend, col=var_name,
373
+ kind=kind, linestyle=linestyle,
374
+ col_wrap=col_wrap, capsize=capsize, **kwargs)
375
+ return f_grid
376
+
377
+
378
+ def dataframe(self, value_name:str='Score',
379
+ var_name:str='Performance',
380
+ alg_legend:str='Algorithm',
381
+ perf_names:str=None):
382
+ """Dataframe"""
383
+ if perf_names is None and isinstance(self.best, np.ndarray):
384
+ func_name = self.statistic_func.__name__
385
+ perf_names = [f'{func_name}({i})'
386
+ for i, k in enumerate(self.best)]
387
+ return dataframe(self, value_name=value_name,
388
+ var_name=var_name,
389
+ alg_legend=alg_legend,
390
+ perf_names=perf_names)
351
391
 
352
392
  @property
353
393
  def n_jobs(self):
@@ -561,7 +601,28 @@ class Difference:
561
601
  values.sort(key=lambda x: self.sorting_func(x[1]))
562
602
  return dict(values)
563
603
 
564
- def plot(self, **kwargs):
604
+ def dataframe(self, value_name:str='Score',
605
+ var_name:str='Best',
606
+ alg_legend:str='Algorithm',
607
+ perf_names:str=None):
608
+ """Dataframe"""
609
+ if perf_names is None and isinstance(self.best, np.ndarray):
610
+ perf_names = [f'{alg}({k})'
611
+ for k, alg in enumerate(self.best)]
612
+ return dataframe(self, value_name=value_name,
613
+ var_name=var_name,
614
+ alg_legend=alg_legend,
615
+ perf_names=perf_names)
616
+
617
+ def plot(self, value_name:str='Difference',
618
+ var_name:str='Best',
619
+ alg_legend:str='Algorithm',
620
+ perf_names:list=None,
621
+ CI:float=0.05,
622
+ kind:str='point', linestyle:str='none',
623
+ col_wrap:int=3, capsize:float=0.2,
624
+ set_refline:bool=True,
625
+ **kwargs):
565
626
  """Plot
566
627
 
567
628
  >>> from sklearn.svm import LinearSVC
@@ -580,5 +641,21 @@ class Difference:
580
641
  >>> diff = perf.difference()
581
642
  >>> diff.plot()
582
643
  """
583
-
584
- return plot_difference(self.statistic_samples, **kwargs)
644
+ import seaborn as sns
645
+ df = self.dataframe(value_name=value_name,
646
+ var_name=var_name,
647
+ alg_legend=alg_legend, perf_names=perf_names)
648
+ title = var_name
649
+ if var_name not in df.columns:
650
+ var_name = None
651
+ col_wrap = None
652
+ ci = lambda x: measurements.CI(x, alpha=CI)
653
+ f_grid = sns.catplot(df, x=value_name, errorbar=ci,
654
+ y=alg_legend, col=var_name,
655
+ kind=kind, linestyle=linestyle,
656
+ col_wrap=col_wrap, capsize=capsize, **kwargs)
657
+ if set_refline:
658
+ f_grid.refline(x=0)
659
+ if isinstance(self.best, str):
660
+ f_grid.facet_axis(0, 0).set_title(f'{title} = {self.best}')
661
+ return f_grid
@@ -23,6 +23,79 @@ import pandas as pd
23
23
  from CompStats.tests.test_performance import DATA
24
24
 
25
25
 
26
+ def test_Difference_dataframe():
27
+ """Test Difference dataframe"""
28
+ from CompStats.metrics import f1_score
29
+
30
+ X, y = load_digits(return_X_y=True)
31
+ _ = train_test_split(X, y, test_size=0.3)
32
+ X_train, X_val, y_train, y_val = _
33
+ ens = RandomForestClassifier().fit(X_train, y_train)
34
+ nb = GaussianNB().fit(X_train, y_train)
35
+ svm = LinearSVC().fit(X_train, y_train)
36
+ score = f1_score(y_val, ens.predict(X_val),
37
+ average=None,
38
+ num_samples=50)
39
+ score(nb.predict(X_val))
40
+ score(svm.predict(X_val))
41
+ diff = score.difference()
42
+ df = diff.dataframe()
43
+ assert 'Best' in df.columns
44
+ score = f1_score(y_val, ens.predict(X_val),
45
+ average='macro',
46
+ num_samples=50)
47
+ score(nb.predict(X_val))
48
+ score(svm.predict(X_val))
49
+ diff = score.difference()
50
+ df = diff.dataframe()
51
+ assert 'Best' not in df.columns
52
+
53
+
54
+ def test_Perf_dataframe():
55
+ """Test Perf dataframe"""
56
+ from CompStats.metrics import f1_score
57
+
58
+ X, y = load_digits(return_X_y=True)
59
+ _ = train_test_split(X, y, test_size=0.3)
60
+ X_train, X_val, y_train, y_val = _
61
+ ens = RandomForestClassifier().fit(X_train, y_train)
62
+ nb = GaussianNB().fit(X_train, y_train)
63
+ svm = LinearSVC().fit(X_train, y_train)
64
+ score = f1_score(y_val, ens.predict(X_val),
65
+ average=None,
66
+ num_samples=50)
67
+ df = score.dataframe()
68
+ score(nb.predict(X_val))
69
+ score(svm.predict(X_val))
70
+ df = score.dataframe()
71
+ assert 'Performance' in df.columns
72
+ score = f1_score(y_val, ens.predict(X_val),
73
+ average='macro',
74
+ num_samples=50)
75
+ score(nb.predict(X_val))
76
+ score(svm.predict(X_val))
77
+ df = score.dataframe()
78
+ assert 'Performance' not in df.columns
79
+
80
+
81
+ def test_Perf_plot_multi():
82
+ """Test Perf plot multiple"""
83
+ from CompStats.metrics import f1_score
84
+
85
+ X, y = load_digits(return_X_y=True)
86
+ _ = train_test_split(X, y, test_size=0.3)
87
+ X_train, X_val, y_train, y_val = _
88
+ ens = RandomForestClassifier().fit(X_train, y_train)
89
+ nb = GaussianNB().fit(X_train, y_train)
90
+ svm = LinearSVC().fit(X_train, y_train)
91
+ score = f1_score(y_val, ens.predict(X_val),
92
+ average=None,
93
+ num_samples=50)
94
+ score(nb.predict(X_val))
95
+ score(svm.predict(X_val))
96
+ f_grid = score.plot()
97
+ assert f_grid is not None
98
+
26
99
  def test_Perf_statistic_one():
27
100
  """Test Perf statistic one alg"""
28
101
  from CompStats.metrics import f1_score
@@ -208,7 +281,7 @@ def test_Difference_plot():
208
281
  diff.plot()
209
282
 
210
283
 
211
- def test_Perf_dataframe():
284
+ def test_Perf_input_dataframe():
212
285
  """Test Perf with dataframe"""
213
286
  from CompStats.interface import Perf
214
287
 
@@ -54,4 +54,29 @@ def metrics_docs(hy_name='y_pred', attr_name='score_func'):
54
54
  return func(*args, **kwargs)
55
55
 
56
56
  return inner
57
- return perf_docs
57
+ return perf_docs
58
+
59
+
60
+ def dataframe(instance, value_name:str='Score',
61
+ var_name:str='Performance',
62
+ alg_legend:str='Algorithm',
63
+ perf_names:list=None):
64
+ """Dataframe"""
65
+ import pandas as pd
66
+ if isinstance(instance.best, str):
67
+ df = pd.DataFrame(dict(instance.statistic_samples.calls.items()))
68
+ return df.melt(var_name=alg_legend,
69
+ value_name=value_name)
70
+ df = pd.DataFrame()
71
+ if not isinstance(instance.statistic, dict):
72
+ iter = instance.statistic_samples.keys()
73
+ else:
74
+ iter = instance.statistic
75
+ for key in iter:
76
+ data = instance.statistic_samples[key]
77
+ _df = pd.DataFrame(data,
78
+ columns=perf_names).melt(value_name=value_name,
79
+ var_name=var_name)
80
+ _df[alg_legend] = key
81
+ df = pd.concat((df, _df))
82
+ return df
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.6
3
+ Version: 0.1.7
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -73,15 +73,12 @@ Once the predictions are available, it is time to measure the algorithm's perfor
73
73
 
74
74
  >>> score = f1_score(y_val, hy, average='macro')
75
75
  >>> score
76
- <Perf(score_func=f1_score)>
77
- Statistic with its standard error (se)
78
- statistic (se)
79
- 0.9332 (0.0113) <= alg-1
76
+ <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
80
77
 
81
78
  The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
82
79
 
83
80
  >>> score.statistic
84
- {'alg-1': 0.9332035615949114}
81
+ 0.9434834454375508
85
82
 
86
83
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
87
84
 
@@ -90,8 +87,8 @@ Continuing with the example, let us assume that one wants to test another classi
90
87
  <Perf(score_func=f1_score)>
91
88
  Statistic with its standard error (se)
92
89
  statistic (se)
93
- 0.9756 (0.0061) <= Random Forest
94
- 0.9332 (0.0113) <= alg-1
90
+ 0.9655 (0.0077) <= Random Forest
91
+ 0.9435 (0.0099) <= alg-1
95
92
 
96
93
  Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
97
94
 
@@ -100,18 +97,18 @@ Let us incorporate another prediction, now with the Naive Bayes classifier, as s
100
97
  <Perf(score_func=f1_score)>
101
98
  Statistic with its standard error (se)
102
99
  statistic (se)
103
- 0.9756 (0.0061) <= Random Forest
104
- 0.9332 (0.0113) <= alg-1
105
- 0.8198 (0.0144) <= Naive Bayes
100
+ 0.9655 (0.0077) <= Random Forest
101
+ 0.9435 (0.0099) <= alg-1
102
+ 0.8549 (0.0153) <= Naive Bayes
106
103
 
107
104
  The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
108
105
 
109
106
  >>> diff = score.difference()
110
107
  >>> diff
111
108
  <Difference>
112
- difference p-values w.r.t Random Forest
113
- 0.0000 <= alg-1
109
+ difference p-values w.r.t Random Forest
114
110
  0.0000 <= Naive Bayes
111
+ 0.0120 <= alg-1
115
112
 
116
113
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
117
114
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.6
3
+ Version: 0.1.7
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -73,15 +73,12 @@ Once the predictions are available, it is time to measure the algorithm's perfor
73
73
 
74
74
  >>> score = f1_score(y_val, hy, average='macro')
75
75
  >>> score
76
- <Perf(score_func=f1_score)>
77
- Statistic with its standard error (se)
78
- statistic (se)
79
- 0.9332 (0.0113) <= alg-1
76
+ <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
80
77
 
81
78
  The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
82
79
 
83
80
  >>> score.statistic
84
- {'alg-1': 0.9332035615949114}
81
+ 0.9434834454375508
85
82
 
86
83
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
87
84
 
@@ -90,8 +87,8 @@ Continuing with the example, let us assume that one wants to test another classi
90
87
  <Perf(score_func=f1_score)>
91
88
  Statistic with its standard error (se)
92
89
  statistic (se)
93
- 0.9756 (0.0061) <= Random Forest
94
- 0.9332 (0.0113) <= alg-1
90
+ 0.9655 (0.0077) <= Random Forest
91
+ 0.9435 (0.0099) <= alg-1
95
92
 
96
93
  Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
97
94
 
@@ -100,18 +97,18 @@ Let us incorporate another prediction, now with the Naive Bayes classifier, as s
100
97
  <Perf(score_func=f1_score)>
101
98
  Statistic with its standard error (se)
102
99
  statistic (se)
103
- 0.9756 (0.0061) <= Random Forest
104
- 0.9332 (0.0113) <= alg-1
105
- 0.8198 (0.0144) <= Naive Bayes
100
+ 0.9655 (0.0077) <= Random Forest
101
+ 0.9435 (0.0099) <= alg-1
102
+ 0.8549 (0.0153) <= Naive Bayes
106
103
 
107
104
  The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
108
105
 
109
106
  >>> diff = score.difference()
110
107
  >>> diff
111
108
  <Difference>
112
- difference p-values w.r.t Random Forest
113
- 0.0000 <= alg-1
109
+ difference p-values w.r.t Random Forest
114
110
  0.0000 <= Naive Bayes
111
+ 0.0120 <= alg-1
115
112
 
116
113
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
117
114
 
@@ -49,15 +49,12 @@ Once the predictions are available, it is time to measure the algorithm's perfor
49
49
 
50
50
  >>> score = f1_score(y_val, hy, average='macro')
51
51
  >>> score
52
- <Perf(score_func=f1_score)>
53
- Statistic with its standard error (se)
54
- statistic (se)
55
- 0.9332 (0.0113) <= alg-1
52
+ <Perf(score_func=f1_score, statistic=0.9435, se=0.0099)>
56
53
 
57
54
  The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
58
55
 
59
56
  >>> score.statistic
60
- {'alg-1': 0.9332035615949114}
57
+ 0.9434834454375508
61
58
 
62
59
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
63
60
 
@@ -66,8 +63,8 @@ Continuing with the example, let us assume that one wants to test another classi
66
63
  <Perf(score_func=f1_score)>
67
64
  Statistic with its standard error (se)
68
65
  statistic (se)
69
- 0.9756 (0.0061) <= Random Forest
70
- 0.9332 (0.0113) <= alg-1
66
+ 0.9655 (0.0077) <= Random Forest
67
+ 0.9435 (0.0099) <= alg-1
71
68
 
72
69
  Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
73
70
 
@@ -76,18 +73,18 @@ Let us incorporate another prediction, now with the Naive Bayes classifier, as s
76
73
  <Perf(score_func=f1_score)>
77
74
  Statistic with its standard error (se)
78
75
  statistic (se)
79
- 0.9756 (0.0061) <= Random Forest
80
- 0.9332 (0.0113) <= alg-1
81
- 0.8198 (0.0144) <= Naive Bayes
76
+ 0.9655 (0.0077) <= Random Forest
77
+ 0.9435 (0.0099) <= alg-1
78
+ 0.8549 (0.0153) <= Naive Bayes
82
79
 
83
80
  The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
84
81
 
85
82
  >>> diff = score.difference()
86
83
  >>> diff
87
84
  <Difference>
88
- difference p-values w.r.t Random Forest
89
- 0.0000 <= alg-1
85
+ difference p-values w.r.t Random Forest
90
86
  0.0000 <= Naive Bayes
87
+ 0.0120 <= alg-1
91
88
 
92
89
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
93
90
 
File without changes
File without changes
File without changes
File without changes
File without changes