CompStats 0.1.5__tar.gz → 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {compstats-0.1.5 → compstats-0.1.6}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.5 → compstats-0.1.6}/CompStats/interface.py +79 -26
  3. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_interface.py +99 -15
  4. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_metrics.py +29 -76
  5. {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/PKG-INFO +20 -17
  6. {compstats-0.1.5 → compstats-0.1.6}/PKG-INFO +20 -17
  7. {compstats-0.1.5 → compstats-0.1.6}/README.rst +19 -16
  8. {compstats-0.1.5 → compstats-0.1.6}/CompStats/bootstrap.py +0 -0
  9. {compstats-0.1.5 → compstats-0.1.6}/CompStats/measurements.py +0 -0
  10. {compstats-0.1.5 → compstats-0.1.6}/CompStats/metrics.py +0 -0
  11. {compstats-0.1.5 → compstats-0.1.6}/CompStats/performance.py +0 -0
  12. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/__init__.py +0 -0
  13. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/data.csv +0 -0
  14. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_bootstrap.py +0 -0
  15. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_measurements.py +0 -0
  16. {compstats-0.1.5 → compstats-0.1.6}/CompStats/tests/test_performance.py +0 -0
  17. {compstats-0.1.5 → compstats-0.1.6}/CompStats/utils.py +0 -0
  18. {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/SOURCES.txt +0 -0
  19. {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/dependency_links.txt +0 -0
  20. {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/requires.txt +0 -0
  21. {compstats-0.1.5 → compstats-0.1.6}/CompStats.egg-info/top_level.txt +0 -0
  22. {compstats-0.1.5 → compstats-0.1.6}/LICENSE +0 -0
  23. {compstats-0.1.5 → compstats-0.1.6}/MANIFEST.in +0 -0
  24. {compstats-0.1.5 → compstats-0.1.6}/pyproject.toml +0 -0
  25. {compstats-0.1.5 → compstats-0.1.6}/setup.cfg +0 -0
  26. {compstats-0.1.5 → compstats-0.1.6}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.5'
14
+ __version__ = '0.1.6'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -147,10 +147,21 @@ class Perf(object):
147
147
  """Prediction statistics with standard error in parenthesis"""
148
148
  arg = 'score_func' if self.error_func is None else 'error_func'
149
149
  func_name = self.statistic_func.__name__
150
- return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
151
-
150
+ statistic = self.statistic
151
+ if isinstance(statistic, dict):
152
+ return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
153
+ elif isinstance(statistic, float):
154
+ return f"<{self.__class__.__name__}({arg}={func_name}, statistic={statistic:0.4f}, se={self.se:0.4f})>"
155
+ desc = [f'{k:0.4f}' for k in statistic]
156
+ desc = ', '.join(desc)
157
+ desc_se = [f'{k:0.4f}' for k in self.se]
158
+ desc_se = ', '.join(desc_se)
159
+ return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
160
+
152
161
  def __str__(self):
153
162
  """Prediction statistics with standard error in parenthesis"""
163
+ if not isinstance(self.statistic, dict):
164
+ return self.__repr__()
154
165
 
155
166
  se = self.se
156
167
  output = ["Statistic with its standard error (se)"]
@@ -173,6 +184,7 @@ class Perf(object):
173
184
  if k == 0:
174
185
  k = 1
175
186
  name = f'alg-{k}'
187
+ self.best = None
176
188
  self.predictions[name] = np.asanyarray(y_pred)
177
189
  samples = self._statistic_samples
178
190
  calls = samples.calls
@@ -180,7 +192,7 @@ class Perf(object):
180
192
  del calls[name]
181
193
  return self
182
194
 
183
- def difference(self, wrt_to: str=None):
195
+ def difference(self, wrt: str=None):
184
196
  """Compute the difference w.r.t any algorithm by default is the best
185
197
 
186
198
  >>> from sklearn.svm import LinearSVC
@@ -201,37 +213,53 @@ class Perf(object):
201
213
  difference p-values w.r.t alg-1
202
214
  forest 0.06
203
215
  """
204
- if wrt_to is None:
205
- wrt_to = self.best[0]
206
- base = self.statistic_samples.calls[wrt_to]
216
+ if wrt is None:
217
+ wrt = self.best
218
+ if isinstance(wrt, str):
219
+ base = self.statistic_samples.calls[wrt]
220
+ else:
221
+ base = np.array([self.statistic_samples.calls[key][:, col]
222
+ for col, key in enumerate(wrt)]).T
207
223
  sign = 1 if self.statistic_samples.BiB else -1
208
224
  diff = dict()
209
225
  for k, v in self.statistic_samples.calls.items():
210
- if k == wrt_to:
226
+ if base.ndim == 1 and k == wrt:
211
227
  continue
212
228
  diff[k] = sign * (base - v)
213
229
  diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
214
- statistic=self.statistic,
215
- best=self.best[0])
230
+ statistic=self.statistic)
216
231
  diff_ins.sorting_func = self.sorting_func
217
232
  diff_ins.statistic_samples.calls = diff
218
- diff_ins.statistic_samples.info['best'] = self.best[0]
233
+ diff_ins.statistic_samples.info['best'] = self.best
234
+ diff_ins.best = self.best
219
235
  return diff_ins
220
236
 
221
237
  @property
222
238
  def best(self):
223
- """Best system"""
224
-
225
- try:
239
+ """System with best performance"""
240
+ if hasattr(self, '_best') and self._best is not None:
226
241
  return self._best
227
- except AttributeError:
228
- statistic = [(k, v) for k, v in self.statistic.items()]
229
- statistic = sorted(statistic,
230
- key=lambda x: self.sorting_func(x[1]),
231
- reverse=self.statistic_samples.BiB)
232
- self._best = statistic[0]
242
+ BiB = True if self.statistic_samples.BiB else False
243
+ keys = np.array(list(self.statistic.keys()))
244
+ data = np.asanyarray([self.statistic[k]
245
+ for k in keys])
246
+ if isinstance(self.statistic[keys[0]], np.ndarray):
247
+ if BiB:
248
+ best = data.argmax(axis=0)
249
+ else:
250
+ best = data.argmin(axis=0)
251
+ else:
252
+ if BiB:
253
+ best = data.argmax()
254
+ else:
255
+ best = data.argmin()
256
+ self._best = keys[best]
233
257
  return self._best
234
258
 
259
+ @best.setter
260
+ def best(self, value):
261
+ self._best = value
262
+
235
263
  @property
236
264
  def sorting_func(self):
237
265
  """Rank systems when multiple performances are used"""
@@ -265,6 +293,8 @@ class Perf(object):
265
293
  for k, v in self.predictions.items()],
266
294
  key=lambda x: self.sorting_func(x[1]),
267
295
  reverse=self.statistic_samples.BiB)
296
+ if len(data) == 1:
297
+ return data[0][1]
268
298
  return dict(data)
269
299
 
270
300
  @property
@@ -287,7 +317,10 @@ class Perf(object):
287
317
  {'alg-1': 0.0, 'forest': 0.026945730782184187}
288
318
  """
289
319
 
290
- return SE(self.statistic_samples)
320
+ output = SE(self.statistic_samples)
321
+ if len(output) == 1:
322
+ return list(output.values())[0]
323
+ return output
291
324
 
292
325
  def plot(self, **kwargs):
293
326
  """plot with seaborn
@@ -439,8 +472,8 @@ class Difference:
439
472
  """
440
473
 
441
474
  statistic_samples:StatisticSamples=None
442
- best:str=None
443
475
  statistic:dict=None
476
+ best:str=None
444
477
 
445
478
  @property
446
479
  def sorting_func(self):
@@ -453,11 +486,19 @@ class Difference:
453
486
 
454
487
  def __repr__(self):
455
488
  """p-value"""
456
- return f"<{self.__class__.__name__}>\n{self}"
489
+ return f"<{self.__class__.__name__}>\n{self}"
457
490
 
458
491
  def __str__(self):
459
492
  """p-value"""
460
- output = [f"difference p-values w.r.t {self.best}"]
493
+ if isinstance(self.best, str):
494
+ best = f' w.r.t {self.best}'
495
+ else:
496
+ best = ''
497
+ output = [f"difference p-values {best}"]
498
+ best = self.best
499
+ if isinstance(best, np.ndarray):
500
+ desc = ', '.join(best)
501
+ output.append(f'{desc} <= Best')
461
502
  for key, value in self.p_value().items():
462
503
  if isinstance(value, float):
463
504
  output.append(f'{value:0.4f} <= {key}')
@@ -465,8 +506,20 @@ class Difference:
465
506
  desc = [f'{v:0.4f}' for v in value]
466
507
  desc = ', '.join(desc)
467
508
  desc = f'{desc} <= {key}'
509
+ output.append(desc)
468
510
  return "\n".join(output)
469
511
 
512
+ def _delta_best(self):
513
+ """Compute multiple delta"""
514
+ if isinstance(self.best, str):
515
+ return self.statistic[self.best]
516
+ keys = np.unique(self.best)
517
+ statistic = np.array([self.statistic[k]
518
+ for k in keys])
519
+ m = {v: k for k, v in enumerate(keys)}
520
+ best = np.array([m[x] for x in self.best])
521
+ return statistic[best, np.arange(best.shape[0])]
522
+
470
523
  def p_value(self, right:bool=True):
471
524
  """Compute p_value of the differences
472
525
 
@@ -492,10 +545,10 @@ class Difference:
492
545
  """
493
546
  values = []
494
547
  sign = 1 if self.statistic_samples.BiB else -1
495
- ndim = self.statistic[self.best].ndim
548
+ delta_best = self._delta_best()
496
549
  for k, v in self.statistic_samples.calls.items():
497
- delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
498
- if ndim == 0:
550
+ delta = 2 * sign * (delta_best - self.statistic[k])
551
+ if not isinstance(delta_best, np.ndarray):
499
552
  if right:
500
553
  values.append((k, (v >= delta).mean()))
501
554
  else:
@@ -16,12 +16,109 @@ import numpy as np
16
16
  from sklearn.base import clone
17
17
  from sklearn.svm import LinearSVC
18
18
  from sklearn.ensemble import RandomForestClassifier
19
- from sklearn.datasets import load_iris
19
+ from sklearn.naive_bayes import GaussianNB
20
+ from sklearn.datasets import load_iris, load_digits
20
21
  from sklearn.model_selection import train_test_split
21
22
  import pandas as pd
22
23
  from CompStats.tests.test_performance import DATA
23
24
 
24
25
 
26
+ def test_Perf_statistic_one():
27
+ """Test Perf statistic one alg"""
28
+ from CompStats.metrics import f1_score
29
+
30
+ X, y = load_digits(return_X_y=True)
31
+ _ = train_test_split(X, y, test_size=0.3)
32
+ X_train, X_val, y_train, y_val = _
33
+ ens = RandomForestClassifier().fit(X_train, y_train)
34
+ nb = GaussianNB().fit(X_train, y_train)
35
+ svm = LinearSVC().fit(X_train, y_train)
36
+ score = f1_score(y_val, ens.predict(X_val),
37
+ average=None,
38
+ num_samples=50)
39
+ assert isinstance(score.statistic, np.ndarray)
40
+ assert isinstance(str(score), str)
41
+ score = f1_score(y_val, ens.predict(X_val),
42
+ average='macro',
43
+ num_samples=50)
44
+ assert isinstance(score.statistic, float)
45
+ assert isinstance(str(score), str)
46
+ assert isinstance(score.se, float)
47
+
48
+ def test_Perf_best():
49
+ """Test Perf best"""
50
+ from CompStats.metrics import f1_score
51
+
52
+ X, y = load_digits(return_X_y=True)
53
+ _ = train_test_split(X, y, test_size=0.3)
54
+ X_train, X_val, y_train, y_val = _
55
+ ens = RandomForestClassifier().fit(X_train, y_train)
56
+ nb = GaussianNB().fit(X_train, y_train)
57
+ svm = LinearSVC().fit(X_train, y_train)
58
+ score = f1_score(y_val, average=None,
59
+ num_samples=50)
60
+ score(ens.predict(X_val), name='forest')
61
+ score(nb.predict(X_val), name='NB')
62
+ score(svm.predict(X_val), name='svm')
63
+ assert isinstance(score.best, np.ndarray)
64
+ score = f1_score(y_val, average='macro',
65
+ num_samples=50)
66
+ score(ens.predict(X_val), name='forest')
67
+ score(nb.predict(X_val), name='NB')
68
+ score(svm.predict(X_val), name='svm')
69
+ assert isinstance(score.best, str)
70
+
71
+
72
+ def test_difference_best():
73
+ """Test multiple performance measures"""
74
+ from CompStats.metrics import f1_score
75
+
76
+ X, y = load_digits(return_X_y=True)
77
+ _ = train_test_split(X, y, test_size=0.3)
78
+ X_train, X_val, y_train, y_val = _
79
+ ens = RandomForestClassifier().fit(X_train, y_train)
80
+ nb = GaussianNB().fit(X_train, y_train)
81
+ svm = LinearSVC().fit(X_train, y_train)
82
+ score = f1_score(y_val, average=None,
83
+ num_samples=50)
84
+ score(ens.predict(X_val), name='forest')
85
+ score(nb.predict(X_val), name='NB')
86
+ score(svm.predict(X_val), name='svm')
87
+ diff = score.difference()
88
+ assert isinstance(diff.best, np.ndarray)
89
+ score = f1_score(y_val, average='macro',
90
+ num_samples=50)
91
+ score(ens.predict(X_val), name='forest')
92
+ score(nb.predict(X_val), name='NB')
93
+ score(svm.predict(X_val), name='svm')
94
+ diff = score.difference()
95
+ assert isinstance(diff.best, str)
96
+
97
+
98
+ def test_difference_str__():
99
+ """Test f1_score"""
100
+ from CompStats.metrics import f1_score
101
+
102
+ X, y = load_iris(return_X_y=True)
103
+ _ = train_test_split(X, y, test_size=0.3)
104
+ X_train, X_val, y_train, y_val = _
105
+ ens = RandomForestClassifier().fit(X_train, y_train)
106
+ nb = GaussianNB().fit(X_train, y_train)
107
+ perf = f1_score(y_val, nb.predict(X_val),
108
+ forest=ens.predict(X_val),
109
+ num_samples=50, average=None)
110
+ diff = perf.difference()
111
+ p_values = diff.p_value(right=False)
112
+ dd = list(p_values.values())[0]
113
+ assert isinstance(dd, np.ndarray)
114
+ for average in ['macro', None]:
115
+ perf = f1_score(y_val, nb.predict(X_val),
116
+ forest=ens.predict(X_val),
117
+ num_samples=50, average=average)
118
+ diff = perf.difference()
119
+ print(diff)
120
+
121
+
25
122
  def test_Perf():
26
123
  """Test perf"""
27
124
  from CompStats.interface import Perf
@@ -47,6 +144,7 @@ def test_Perf_statistic():
47
144
  X_train, X_val, y_train, y_val = _
48
145
  ens = RandomForestClassifier().fit(X_train, y_train)
49
146
  perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
147
+ perf(ens.predict(X_val))
50
148
  assert 'forest' in perf.statistic
51
149
 
52
150
 
@@ -79,20 +177,6 @@ def test_Perf_clone():
79
177
  assert np.all(samples == perf2.statistic_samples._samples)
80
178
 
81
179
 
82
- def test_Perf_best():
83
- """Test Perf.best"""
84
- from CompStats.interface import Perf
85
-
86
- X, y = load_iris(return_X_y=True)
87
- _ = train_test_split(X, y, test_size=0.3)
88
- X_train, X_val, y_train, y_val = _
89
- m = LinearSVC().fit(X_train, y_train)
90
- hy = m.predict(X_val)
91
- ens = RandomForestClassifier().fit(X_train, y_train)
92
- perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
93
- assert len(perf.best) == 2
94
-
95
-
96
180
  def test_Perf_difference():
97
181
  """Test difference"""
98
182
  from CompStats.interface import Perf, Difference
@@ -11,8 +11,6 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- import numpy as np
15
- from sklearn.svm import LinearSVC
16
14
  from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
17
15
  from sklearn.naive_bayes import GaussianNB
18
16
  from sklearn.datasets import load_iris, load_diabetes
@@ -20,24 +18,6 @@ from sklearn.model_selection import train_test_split
20
18
  from sklearn import metrics
21
19
 
22
20
 
23
- def test_difference_f1_score():
24
- """Test f1_score"""
25
- from CompStats.metrics import f1_score
26
-
27
- X, y = load_iris(return_X_y=True)
28
- _ = train_test_split(X, y, test_size=0.3)
29
- X_train, X_val, y_train, y_val = _
30
- ens = RandomForestClassifier().fit(X_train, y_train)
31
- nb = GaussianNB().fit(X_train, y_train)
32
- perf = f1_score(y_val, nb.predict(X_val),
33
- forest=ens.predict(X_val),
34
- num_samples=50, average=None)
35
- diff = perf.difference()
36
- p_values = diff.p_value(right=False)
37
- dd = list(p_values.values())[0]
38
- assert isinstance(dd, np.ndarray)
39
- print(diff)
40
-
41
21
  def test_f1_score():
42
22
  """Test f1_score"""
43
23
  from CompStats.metrics import f1_score
@@ -49,9 +29,9 @@ def test_f1_score():
49
29
  hy = ens.predict(X_val)
50
30
  perf = f1_score(y_val, forest=hy,
51
31
  num_samples=50, average='macro')
52
- assert 'forest' in perf.statistic
32
+ assert isinstance(perf.statistic, float)
53
33
  _ = metrics.f1_score(y_val, hy, average='macro')
54
- assert _ == perf.statistic['forest']
34
+ assert _ == perf.statistic
55
35
  perf = f1_score(y_val, hy, average=None)
56
36
  assert str(perf) is not None
57
37
  nb = GaussianNB().fit(X_train, y_train)
@@ -70,9 +50,8 @@ def test_accuracy_score():
70
50
  hy = ens.predict(X_val)
71
51
  perf = accuracy_score(y_val, forest=hy,
72
52
  num_samples=50)
73
- assert 'forest' in perf.statistic
74
53
  _ = metrics.accuracy_score(y_val, hy)
75
- assert _ == perf.statistic['forest']
54
+ assert _ == perf.statistic
76
55
 
77
56
 
78
57
  def test_balanced_accuracy_score():
@@ -86,9 +65,8 @@ def test_balanced_accuracy_score():
86
65
  hy = ens.predict(X_val)
87
66
  perf = balanced_accuracy_score(y_val, forest=hy,
88
67
  num_samples=50)
89
- assert 'forest' in perf.statistic
90
68
  _ = metrics.balanced_accuracy_score(y_val, hy)
91
- assert _ == perf.statistic['forest']
69
+ assert _ == perf.statistic
92
70
 
93
71
 
94
72
  def test_top_k_accuracy_score():
@@ -103,9 +81,8 @@ def test_top_k_accuracy_score():
103
81
  perf = top_k_accuracy_score(y_val,
104
82
  forest=hy,
105
83
  num_samples=50)
106
- assert 'forest' in perf.statistic
107
84
  _ = metrics.top_k_accuracy_score(y_val, hy)
108
- assert _ == perf.statistic['forest']
85
+ assert _ == perf.statistic
109
86
 
110
87
 
111
88
  def test_average_precision_score():
@@ -120,9 +97,8 @@ def test_average_precision_score():
120
97
  perf = average_precision_score(y_val,
121
98
  forest=hy,
122
99
  num_samples=50)
123
- assert 'forest' in perf.statistic
124
100
  _ = metrics.average_precision_score(y_val, hy)
125
- assert _ == perf.statistic['forest']
101
+ assert _ == perf.statistic
126
102
 
127
103
 
128
104
  def test_brier_score_loss():
@@ -138,11 +114,8 @@ def test_brier_score_loss():
138
114
  perf = brier_score_loss(np.where(y_val == 0, 1, 0),
139
115
  forest=hy,
140
116
  num_samples=50)
141
- nb = GaussianNB().fit(X_train, y_train)
142
- perf(nb.predict_proba(X_val)[:, 0])
143
- assert 'forest' in perf.statistic
144
117
  _ = metrics.brier_score_loss(np.where(y_val == 0, 1, 0), hy)
145
- assert _ == perf.statistic['forest']
118
+ assert _ == perf.statistic
146
119
 
147
120
 
148
121
  def test_log_loss():
@@ -158,11 +131,8 @@ def test_log_loss():
158
131
  perf = log_loss(y_val,
159
132
  forest=hy,
160
133
  num_samples=50)
161
- nb = GaussianNB().fit(X_train, y_train)
162
- perf(nb.predict_proba(X_val))
163
- assert 'forest' in perf.statistic
164
134
  _ = metrics.log_loss(y_val, hy)
165
- assert _ == perf.statistic['forest']
135
+ assert _ == perf.statistic
166
136
 
167
137
 
168
138
  def test_precision_score():
@@ -178,9 +148,8 @@ def test_precision_score():
178
148
  perf = precision_score(y_val,
179
149
  forest=hy,
180
150
  num_samples=50, average='macro')
181
- assert 'forest' in perf.statistic
182
151
  _ = metrics.precision_score(y_val, hy, average='macro')
183
- assert _ == perf.statistic['forest']
152
+ assert _ == perf.statistic
184
153
 
185
154
 
186
155
  def test_recall_score():
@@ -196,9 +165,8 @@ def test_recall_score():
196
165
  perf = recall_score(y_val,
197
166
  forest=hy,
198
167
  num_samples=50, average='macro')
199
- assert 'forest' in perf.statistic
200
168
  _ = metrics.recall_score(y_val, hy, average='macro')
201
- assert _ == perf.statistic['forest']
169
+ assert _ == perf.statistic
202
170
 
203
171
 
204
172
  def test_jaccard_score():
@@ -214,9 +182,8 @@ def test_jaccard_score():
214
182
  perf = jaccard_score(y_val,
215
183
  forest=hy,
216
184
  num_samples=50, average='macro')
217
- assert 'forest' in perf.statistic
218
185
  _ = metrics.jaccard_score(y_val, hy, average='macro')
219
- assert _ == perf.statistic['forest']
186
+ assert _ == perf.statistic
220
187
 
221
188
 
222
189
  def test_roc_auc_score():
@@ -232,10 +199,9 @@ def test_roc_auc_score():
232
199
  perf = roc_auc_score(y_val,
233
200
  forest=hy, multi_class='ovr',
234
201
  num_samples=50, average='macro')
235
- assert 'forest' in perf.statistic
236
202
  _ = metrics.roc_auc_score(y_val, hy, multi_class='ovr',
237
203
  average='macro')
238
- assert _ == perf.statistic['forest']
204
+ assert _ == perf.statistic
239
205
 
240
206
 
241
207
  def test_d2_log_loss_score():
@@ -251,9 +217,8 @@ def test_d2_log_loss_score():
251
217
  perf = d2_log_loss_score(y_val,
252
218
  forest=hy,
253
219
  num_samples=50)
254
- assert 'forest' in perf.statistic
255
220
  _ = metrics.d2_log_loss_score(y_val, hy)
256
- assert _ == perf.statistic['forest']
221
+ assert _ == perf.statistic
257
222
 
258
223
 
259
224
  def test_explained_variance_score():
@@ -268,9 +233,8 @@ def test_explained_variance_score():
268
233
  perf = explained_variance_score(y_val,
269
234
  forest=hy,
270
235
  num_samples=50)
271
- assert 'forest' in perf.statistic
272
236
  _ = metrics.explained_variance_score(y_val, hy)
273
- assert _ == perf.statistic['forest']
237
+ assert _ == perf.statistic
274
238
 
275
239
 
276
240
  def test_max_error():
@@ -285,9 +249,8 @@ def test_max_error():
285
249
  perf = max_error(y_val,
286
250
  forest=hy,
287
251
  num_samples=50)
288
- assert 'forest' in perf.statistic
289
252
  _ = metrics.max_error(y_val, hy)
290
- assert _ == perf.statistic['forest']
253
+ assert _ == perf.statistic
291
254
 
292
255
 
293
256
  def test_mean_absolute_error():
@@ -302,9 +265,8 @@ def test_mean_absolute_error():
302
265
  perf = mean_absolute_error(y_val,
303
266
  forest=hy,
304
267
  num_samples=50)
305
- assert 'forest' in perf.statistic
306
268
  _ = metrics.mean_absolute_error(y_val, hy)
307
- assert _ == perf.statistic['forest']
269
+ assert _ == perf.statistic
308
270
 
309
271
 
310
272
  def test_mean_squared_error():
@@ -319,9 +281,8 @@ def test_mean_squared_error():
319
281
  perf = mean_squared_error(y_val,
320
282
  forest=hy,
321
283
  num_samples=50)
322
- assert 'forest' in perf.statistic
323
284
  _ = metrics.mean_squared_error(y_val, hy)
324
- assert _ == perf.statistic['forest']
285
+ assert _ == perf.statistic
325
286
 
326
287
 
327
288
  def test_root_mean_squared_error():
@@ -336,9 +297,8 @@ def test_root_mean_squared_error():
336
297
  perf = root_mean_squared_error(y_val,
337
298
  forest=hy,
338
299
  num_samples=50)
339
- assert 'forest' in perf.statistic
340
300
  _ = metrics.root_mean_squared_error(y_val, hy)
341
- assert _ == perf.statistic['forest']
301
+ assert _ == perf.statistic
342
302
 
343
303
 
344
304
  def test_mean_squared_log_error():
@@ -353,9 +313,8 @@ def test_mean_squared_log_error():
353
313
  perf = mean_squared_log_error(y_val,
354
314
  forest=hy,
355
315
  num_samples=50)
356
- assert 'forest' in perf.statistic
357
316
  _ = metrics.mean_squared_log_error(y_val, hy)
358
- assert _ == perf.statistic['forest']
317
+ assert _ == perf.statistic
359
318
 
360
319
 
361
320
  def test_root_mean_squared_log_error():
@@ -370,9 +329,8 @@ def test_root_mean_squared_log_error():
370
329
  perf = root_mean_squared_log_error(y_val,
371
330
  forest=hy,
372
331
  num_samples=50)
373
- assert 'forest' in perf.statistic
374
332
  _ = metrics.root_mean_squared_log_error(y_val, hy)
375
- assert _ == perf.statistic['forest']
333
+ assert _ == perf.statistic
376
334
 
377
335
 
378
336
  def test_median_absolute_error():
@@ -387,9 +345,8 @@ def test_median_absolute_error():
387
345
  perf = median_absolute_error(y_val,
388
346
  forest=hy,
389
347
  num_samples=50)
390
- assert 'forest' in perf.statistic
391
348
  _ = metrics.median_absolute_error(y_val, hy)
392
- assert _ == perf.statistic['forest']
349
+ assert _ == perf.statistic
393
350
 
394
351
 
395
352
  def test_r2_score():
@@ -402,11 +359,11 @@ def test_r2_score():
402
359
  ens = RandomForestRegressor().fit(X_train, y_train)
403
360
  hy = ens.predict(X_val)
404
361
  perf = r2_score(y_val,
405
- forest=hy,
406
- num_samples=50)
407
- assert 'forest' in perf.statistic
362
+ forest=hy,
363
+ num_samples=50)
364
+
408
365
  _ = metrics.r2_score(y_val, hy)
409
- assert _ == perf.statistic['forest']
366
+ assert _ == perf.statistic
410
367
 
411
368
 
412
369
  def test_mean_poisson_deviance():
@@ -421,9 +378,8 @@ def test_mean_poisson_deviance():
421
378
  perf = mean_poisson_deviance(y_val,
422
379
  forest=hy,
423
380
  num_samples=50)
424
- assert 'forest' in perf.statistic
425
381
  _ = metrics.mean_poisson_deviance(y_val, hy)
426
- assert _ == perf.statistic['forest']
382
+ assert _ == perf.statistic
427
383
 
428
384
 
429
385
  def test_mean_gamma_deviance():
@@ -438,9 +394,8 @@ def test_mean_gamma_deviance():
438
394
  perf = mean_gamma_deviance(y_val,
439
395
  forest=hy,
440
396
  num_samples=50)
441
- assert 'forest' in perf.statistic
442
397
  _ = metrics.mean_gamma_deviance(y_val, hy)
443
- assert _ == perf.statistic['forest']
398
+ assert _ == perf.statistic
444
399
 
445
400
 
446
401
  def test_mean_absolute_percentage_error():
@@ -455,9 +410,8 @@ def test_mean_absolute_percentage_error():
455
410
  perf = mean_absolute_percentage_error(y_val,
456
411
  forest=hy,
457
412
  num_samples=50)
458
- assert 'forest' in perf.statistic
459
413
  _ = metrics.mean_absolute_percentage_error(y_val, hy)
460
- assert _ == perf.statistic['forest']
414
+ assert _ == perf.statistic
461
415
 
462
416
 
463
417
  def test_d2_absolute_error_score():
@@ -472,6 +426,5 @@ def test_d2_absolute_error_score():
472
426
  perf = d2_absolute_error_score(y_val,
473
427
  forest=hy,
474
428
  num_samples=50)
475
- assert 'forest' in perf.statistic
476
429
  _ = metrics.d2_absolute_error_score(y_val, hy)
477
- assert _ == perf.statistic['forest']
430
+ assert _ == perf.statistic
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.5
3
+ Version: 0.1.6
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -73,33 +73,36 @@ Once the predictions are available, it is time to measure the algorithm's perfor
73
73
 
74
74
  >>> score = f1_score(y_val, hy, average='macro')
75
75
  >>> score
76
- <Perf>
77
- Prediction statistics with standard error
78
- alg-1 = 0.936 (0.010)
76
+ <Perf(score_func=f1_score)>
77
+ Statistic with its standard error (se)
78
+ statistic (se)
79
+ 0.9332 (0.0113) <= alg-1
79
80
 
80
81
  The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
81
82
 
82
- >>> score.statistic()
83
- {'alg-1': 0.9355476018466147}
83
+ >>> score.statistic
84
+ {'alg-1': 0.9332035615949114}
84
85
 
85
86
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
86
87
 
87
88
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
88
89
  >>> score(ens.predict(X_val), name='Random Forest')
89
- <Perf>
90
- Prediction statistics with standard error
91
- Random Forest = 0.970 (0.008)
92
- alg-1 = 0.936 (0.010)
90
+ <Perf(score_func=f1_score)>
91
+ Statistic with its standard error (se)
92
+ statistic (se)
93
+ 0.9756 (0.0061) <= Random Forest
94
+ 0.9332 (0.0113) <= alg-1
93
95
 
94
96
  Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
95
97
 
96
98
  >>> nb = GaussianNB().fit(X_train, y_train)
97
99
  >>> score(nb.predict(X_val), name='Naive Bayes')
98
- <Perf>
99
- Prediction statistics with standard error
100
- Random Forest = 0.970 (0.008)
101
- alg-1 = 0.936 (0.010)
102
- Naive Bayes = 0.821 (0.016)
100
+ <Perf(score_func=f1_score)>
101
+ Statistic with its standard error (se)
102
+ statistic (se)
103
+ 0.9756 (0.0061) <= Random Forest
104
+ 0.9332 (0.0113) <= alg-1
105
+ 0.8198 (0.0144) <= Naive Bayes
103
106
 
104
107
  The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
105
108
 
@@ -107,8 +110,8 @@ The final step is to compare the performance of the three classifiers, which can
107
110
  >>> diff
108
111
  <Difference>
109
112
  difference p-values w.r.t Random Forest
110
- alg-1 0.0
111
- Naive Bayes 0.0
113
+ 0.0000 <= alg-1
114
+ 0.0000 <= Naive Bayes
112
115
 
113
116
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
114
117
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.5
3
+ Version: 0.1.6
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -73,33 +73,36 @@ Once the predictions are available, it is time to measure the algorithm's perfor
73
73
 
74
74
  >>> score = f1_score(y_val, hy, average='macro')
75
75
  >>> score
76
- <Perf>
77
- Prediction statistics with standard error
78
- alg-1 = 0.936 (0.010)
76
+ <Perf(score_func=f1_score)>
77
+ Statistic with its standard error (se)
78
+ statistic (se)
79
+ 0.9332 (0.0113) <= alg-1
79
80
 
80
81
  The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
81
82
 
82
- >>> score.statistic()
83
- {'alg-1': 0.9355476018466147}
83
+ >>> score.statistic
84
+ {'alg-1': 0.9332035615949114}
84
85
 
85
86
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
86
87
 
87
88
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
88
89
  >>> score(ens.predict(X_val), name='Random Forest')
89
- <Perf>
90
- Prediction statistics with standard error
91
- Random Forest = 0.970 (0.008)
92
- alg-1 = 0.936 (0.010)
90
+ <Perf(score_func=f1_score)>
91
+ Statistic with its standard error (se)
92
+ statistic (se)
93
+ 0.9756 (0.0061) <= Random Forest
94
+ 0.9332 (0.0113) <= alg-1
93
95
 
94
96
  Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
95
97
 
96
98
  >>> nb = GaussianNB().fit(X_train, y_train)
97
99
  >>> score(nb.predict(X_val), name='Naive Bayes')
98
- <Perf>
99
- Prediction statistics with standard error
100
- Random Forest = 0.970 (0.008)
101
- alg-1 = 0.936 (0.010)
102
- Naive Bayes = 0.821 (0.016)
100
+ <Perf(score_func=f1_score)>
101
+ Statistic with its standard error (se)
102
+ statistic (se)
103
+ 0.9756 (0.0061) <= Random Forest
104
+ 0.9332 (0.0113) <= alg-1
105
+ 0.8198 (0.0144) <= Naive Bayes
103
106
 
104
107
  The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
105
108
 
@@ -107,8 +110,8 @@ The final step is to compare the performance of the three classifiers, which can
107
110
  >>> diff
108
111
  <Difference>
109
112
  difference p-values w.r.t Random Forest
110
- alg-1 0.0
111
- Naive Bayes 0.0
113
+ 0.0000 <= alg-1
114
+ 0.0000 <= Naive Bayes
112
115
 
113
116
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
114
117
 
@@ -49,33 +49,36 @@ Once the predictions are available, it is time to measure the algorithm's perfor
49
49
 
50
50
  >>> score = f1_score(y_val, hy, average='macro')
51
51
  >>> score
52
- <Perf>
53
- Prediction statistics with standard error
54
- alg-1 = 0.936 (0.010)
52
+ <Perf(score_func=f1_score)>
53
+ Statistic with its standard error (se)
54
+ statistic (se)
55
+ 0.9332 (0.0113) <= alg-1
55
56
 
56
57
  The previous code shows the macro-f1 score and, in parenthesis, its standard error. The actual performance value is stored in the `statistic` function.
57
58
 
58
- >>> score.statistic()
59
- {'alg-1': 0.9355476018466147}
59
+ >>> score.statistic
60
+ {'alg-1': 0.9332035615949114}
60
61
 
61
62
  Continuing with the example, let us assume that one wants to test another classifier on the same problem, in this case, a random forest, as can be seen in the following two lines. The second line predicts the validation set and sets it to the analysis.
62
63
 
63
64
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
64
65
  >>> score(ens.predict(X_val), name='Random Forest')
65
- <Perf>
66
- Prediction statistics with standard error
67
- Random Forest = 0.970 (0.008)
68
- alg-1 = 0.936 (0.010)
66
+ <Perf(score_func=f1_score)>
67
+ Statistic with its standard error (se)
68
+ statistic (se)
69
+ 0.9756 (0.0061) <= Random Forest
70
+ 0.9332 (0.0113) <= alg-1
69
71
 
70
72
  Let us incorporate another prediction, now with the Naive Bayes classifier, as seen below.
71
73
 
72
74
  >>> nb = GaussianNB().fit(X_train, y_train)
73
75
  >>> score(nb.predict(X_val), name='Naive Bayes')
74
- <Perf>
75
- Prediction statistics with standard error
76
- Random Forest = 0.970 (0.008)
77
- alg-1 = 0.936 (0.010)
78
- Naive Bayes = 0.821 (0.016)
76
+ <Perf(score_func=f1_score)>
77
+ Statistic with its standard error (se)
78
+ statistic (se)
79
+ 0.9756 (0.0061) <= Random Forest
80
+ 0.9332 (0.0113) <= alg-1
81
+ 0.8198 (0.0144) <= Naive Bayes
79
82
 
80
83
  The final step is to compare the performance of the three classifiers, which can be done with the `difference` method, as seen next.
81
84
 
@@ -83,8 +86,8 @@ The final step is to compare the performance of the three classifiers, which can
83
86
  >>> diff
84
87
  <Difference>
85
88
  difference p-values w.r.t Random Forest
86
- alg-1 0.0
87
- Naive Bayes 0.0
89
+ 0.0000 <= alg-1
90
+ 0.0000 <= Naive Bayes
88
91
 
89
92
  The class `Difference` has the `plot` method that can be used to depict the difference with respect to the best.
90
93
 
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes