CompStats 0.1.3__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {compstats-0.1.3 → compstats-0.1.5}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.3 → compstats-0.1.5}/CompStats/interface.py +90 -33
  3. compstats-0.1.5/CompStats/metrics.py +599 -0
  4. {compstats-0.1.3 → compstats-0.1.5}/CompStats/performance.py +29 -20
  5. {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_interface.py +2 -2
  6. compstats-0.1.5/CompStats/tests/test_metrics.py +477 -0
  7. compstats-0.1.5/CompStats/utils.py +57 -0
  8. {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/PKG-INFO +67 -2
  9. {compstats-0.1.3 → compstats-0.1.5}/PKG-INFO +67 -2
  10. compstats-0.1.5/README.rst +93 -0
  11. compstats-0.1.3/CompStats/metrics.py +0 -197
  12. compstats-0.1.3/CompStats/tests/test_metrics.py +0 -56
  13. compstats-0.1.3/CompStats/utils.py +0 -25
  14. compstats-0.1.3/README.rst +0 -28
  15. {compstats-0.1.3 → compstats-0.1.5}/CompStats/bootstrap.py +0 -0
  16. {compstats-0.1.3 → compstats-0.1.5}/CompStats/measurements.py +0 -0
  17. {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/__init__.py +0 -0
  18. {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/data.csv +0 -0
  19. {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_bootstrap.py +0 -0
  20. {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_measurements.py +0 -0
  21. {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_performance.py +0 -0
  22. {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/SOURCES.txt +0 -0
  23. {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/dependency_links.txt +0 -0
  24. {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/requires.txt +0 -0
  25. {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/top_level.txt +0 -0
  26. {compstats-0.1.3 → compstats-0.1.5}/LICENSE +0 -0
  27. {compstats-0.1.3 → compstats-0.1.5}/MANIFEST.in +0 -0
  28. {compstats-0.1.3 → compstats-0.1.5}/pyproject.toml +0 -0
  29. {compstats-0.1.3 → compstats-0.1.5}/setup.cfg +0 -0
  30. {compstats-0.1.3 → compstats-0.1.5}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.3'
14
+ __version__ = '0.1.5'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -31,8 +31,8 @@ class Perf(object):
31
31
  :type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
32
32
  :param error_func: Function to measure the performance where the best algorithm has the lowest value.
33
33
  :type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
34
- :param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
35
- :type args: numpy.ndarray
34
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
35
+ :type y_pred: numpy.ndarray
36
36
  :param kwargs: Predictions, the algorithms will be identified using the keyword
37
37
  :type kwargs: numpy.ndarray
38
38
  :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
@@ -59,19 +59,21 @@ class Perf(object):
59
59
  >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
60
60
  >>> perf
61
61
  <Perf>
62
- Prediction statistics with standard error
63
- alg-1 = 1.000 (0.000)
64
- forest = 0.946 (0.038)
62
+ Statistic with its standard error (se)
63
+ statistic (se)
64
+ 0.9792 (0.0221) <= alg-1
65
+ 0.9744 (0.0246) <= forest
65
66
 
66
67
  If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
67
68
 
68
69
  >>> lr = LogisticRegression().fit(X_train, y_train)
69
70
  >>> perf(lr.predict(X_val), name='Log. Reg.')
70
71
  <Perf>
71
- Prediction statistics with standard error
72
- alg-1 = 1.000 (0.000)
73
- forest = 0.946 (0.038)
74
- Log. Reg. = 0.946 (0.038)
72
+ Statistic with its standard error (se)
73
+ statistic (se)
74
+ 1.0000 (0.0000) <= Log. Reg.
75
+ 0.9792 (0.0221) <= alg-1
76
+ 0.9744 (0.0246) <= forest
75
77
 
76
78
  The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
77
79
 
@@ -79,13 +81,14 @@ class Perf(object):
79
81
  >>> perf_error.error_func = lambda y, hy: (y != hy).mean()
80
82
  >>> perf_error
81
83
  <Perf>
82
- Prediction statistics with standard error
83
- alg-1 = 0.000 (0.000)
84
- forest = 0.044 (0.030)
85
- Log. Reg. = 0.044 (0.030)
84
+ Statistic with its standard error (se)
85
+ statistic (se)
86
+ 0.0000 (0.0000) <= Log. Reg.
87
+ 0.0222 (0.0237) <= alg-1
88
+ 0.0222 (0.0215) <= forest
86
89
 
87
90
  """
88
- def __init__(self, y_true, *args,
91
+ def __init__(self, y_true, *y_pred,
89
92
  score_func=balanced_accuracy_score,
90
93
  error_func=None,
91
94
  num_samples: int=500,
@@ -96,7 +99,7 @@ class Perf(object):
96
99
  self.score_func = score_func
97
100
  self.error_func = error_func
98
101
  algs = {}
99
- for k, v in enumerate(args):
102
+ for k, v in enumerate(y_pred):
100
103
  algs[f'alg-{k+1}'] = np.asanyarray(v)
101
104
  algs.update(**kwargs)
102
105
  self.predictions = algs
@@ -104,6 +107,7 @@ class Perf(object):
104
107
  self.num_samples = num_samples
105
108
  self.n_jobs = n_jobs
106
109
  self.use_tqdm = use_tqdm
110
+ self.sorting_func = np.linalg.norm
107
111
  self._init()
108
112
 
109
113
  def _init(self):
@@ -136,19 +140,30 @@ class Perf(object):
136
140
  ins = klass(**params)
137
141
  ins.predictions = dict(self.predictions)
138
142
  ins._statistic_samples._samples = self.statistic_samples._samples
143
+ ins.sorting_func = self.sorting_func
139
144
  return ins
140
-
145
+
141
146
  def __repr__(self):
142
147
  """Prediction statistics with standard error in parenthesis"""
143
- return f"<{self.__class__.__name__}>\n{self}"
148
+ arg = 'score_func' if self.error_func is None else 'error_func'
149
+ func_name = self.statistic_func.__name__
150
+ return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
144
151
 
145
152
  def __str__(self):
146
153
  """Prediction statistics with standard error in parenthesis"""
147
154
 
148
- se = self.se()
149
- output = ["Prediction statistics with standard error"]
150
- for key, value in self.statistic().items():
151
- output.append(f'{key} = {value:0.3f} ({se[key]:0.3f})')
155
+ se = self.se
156
+ output = ["Statistic with its standard error (se)"]
157
+ output.append("statistic (se)")
158
+ for key, value in self.statistic.items():
159
+ if isinstance(value, float):
160
+ desc = f'{value:0.4f} ({se[key]:0.4f}) <= {key}'
161
+ else:
162
+ desc = [f'{v:0.4f} ({k:0.4f})'
163
+ for v, k in zip(value, se[key])]
164
+ desc = ', '.join(desc)
165
+ desc = f'{desc} <= {key}'
166
+ output.append(desc)
152
167
  return "\n".join(output)
153
168
 
154
169
  def __call__(self, y_pred, name=None):
@@ -196,8 +211,9 @@ class Perf(object):
196
211
  continue
197
212
  diff[k] = sign * (base - v)
198
213
  diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
199
- statistic=self.statistic(),
214
+ statistic=self.statistic,
200
215
  best=self.best[0])
216
+ diff_ins.sorting_func = self.sorting_func
201
217
  diff_ins.statistic_samples.calls = diff
202
218
  diff_ins.statistic_samples.info['best'] = self.best[0]
203
219
  return diff_ins
@@ -209,12 +225,23 @@ class Perf(object):
209
225
  try:
210
226
  return self._best
211
227
  except AttributeError:
212
- statistic = [(k, v) for k, v in self.statistic().items()]
213
- statistic = sorted(statistic, key=lambda x: x[1],
228
+ statistic = [(k, v) for k, v in self.statistic.items()]
229
+ statistic = sorted(statistic,
230
+ key=lambda x: self.sorting_func(x[1]),
214
231
  reverse=self.statistic_samples.BiB)
215
232
  self._best = statistic[0]
216
233
  return self._best
234
+
235
+ @property
236
+ def sorting_func(self):
237
+ """Rank systems when multiple performances are used"""
238
+ return self._sorting_func
239
+
240
+ @sorting_func.setter
241
+ def sorting_func(self, value):
242
+ self._sorting_func = value
217
243
 
244
+ @property
218
245
  def statistic(self):
219
246
  """Statistic
220
247
 
@@ -230,15 +257,17 @@ class Perf(object):
230
257
  >>> hy = m.predict(X_val)
231
258
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
232
259
  >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
233
- >>> perf.statistic()
260
+ >>> perf.statistic
234
261
  {'alg-1': 1.0, 'forest': 0.9500891265597148}
235
262
  """
236
263
 
237
264
  data = sorted([(k, self.statistic_func(self.y_true, v))
238
265
  for k, v in self.predictions.items()],
239
- key=lambda x: x[1], reverse=self.statistic_samples.BiB)
266
+ key=lambda x: self.sorting_func(x[1]),
267
+ reverse=self.statistic_samples.BiB)
240
268
  return dict(data)
241
269
 
270
+ @property
242
271
  def se(self):
243
272
  """Standard Error
244
273
 
@@ -254,9 +283,10 @@ class Perf(object):
254
283
  >>> hy = m.predict(X_val)
255
284
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
256
285
  >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
257
- >>> print(perf.se())
286
+ >>> perf.se
258
287
  {'alg-1': 0.0, 'forest': 0.026945730782184187}
259
288
  """
289
+
260
290
  return SE(self.statistic_samples)
261
291
 
262
292
  def plot(self, **kwargs):
@@ -405,13 +435,22 @@ class Difference:
405
435
  >>> diff
406
436
  <Difference>
407
437
  difference p-values w.r.t alg-1
408
- forest 0.3
438
+ 0.0780 <= forest
409
439
  """
410
440
 
411
441
  statistic_samples:StatisticSamples=None
412
442
  best:str=None
413
443
  statistic:dict=None
414
444
 
445
+ @property
446
+ def sorting_func(self):
447
+ """Rank systems when multiple performances are used"""
448
+ return self._sorting_func
449
+
450
+ @sorting_func.setter
451
+ def sorting_func(self, value):
452
+ self._sorting_func = value
453
+
415
454
  def __repr__(self):
416
455
  """p-value"""
417
456
  return f"<{self.__class__.__name__}>\n{self}"
@@ -419,12 +458,20 @@ class Difference:
419
458
  def __str__(self):
420
459
  """p-value"""
421
460
  output = [f"difference p-values w.r.t {self.best}"]
422
- for k, v in self.p_value().items():
423
- output.append(f'{k} {v}')
461
+ for key, value in self.p_value().items():
462
+ if isinstance(value, float):
463
+ output.append(f'{value:0.4f} <= {key}')
464
+ else:
465
+ desc = [f'{v:0.4f}' for v in value]
466
+ desc = ', '.join(desc)
467
+ desc = f'{desc} <= {key}'
424
468
  return "\n".join(output)
425
469
 
426
- def p_value(self):
470
+ def p_value(self, right:bool=True):
427
471
  """Compute p_value of the differences
472
+
473
+ :param right: Estimate the p-value using :math:`\\text{sample} \\geq 2\\delta`
474
+ :type right: bool
428
475
 
429
476
  >>> from sklearn.svm import LinearSVC
430
477
  >>> from sklearn.ensemble import RandomForestClassifier
@@ -445,10 +492,20 @@ class Difference:
445
492
  """
446
493
  values = []
447
494
  sign = 1 if self.statistic_samples.BiB else -1
495
+ ndim = self.statistic[self.best].ndim
448
496
  for k, v in self.statistic_samples.calls.items():
449
497
  delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
450
- values.append((k, (v > delta).mean()))
451
- values.sort(key=lambda x: x[1])
498
+ if ndim == 0:
499
+ if right:
500
+ values.append((k, (v >= delta).mean()))
501
+ else:
502
+ values.append((k, (v <= 0).mean()))
503
+ else:
504
+ if right:
505
+ values.append((k, (v >= delta).mean(axis=0)))
506
+ else:
507
+ values.append((k, (v <= 0).mean(axis=0)))
508
+ values.sort(key=lambda x: self.sorting_func(x[1]))
452
509
  return dict(values)
453
510
 
454
511
  def plot(self, **kwargs):