CompStats 0.1.4__tar.gz → 0.1.6__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {compstats-0.1.4 → compstats-0.1.6}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.4 → compstats-0.1.6}/CompStats/interface.py +133 -30
  3. {compstats-0.1.4 → compstats-0.1.6}/CompStats/metrics.py +75 -29
  4. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_interface.py +99 -15
  5. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_metrics.py +34 -57
  6. compstats-0.1.6/CompStats/utils.py +57 -0
  7. {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/PKG-INFO +20 -17
  8. {compstats-0.1.4 → compstats-0.1.6}/PKG-INFO +20 -17
  9. {compstats-0.1.4 → compstats-0.1.6}/README.rst +19 -16
  10. compstats-0.1.4/CompStats/utils.py +0 -55
  11. {compstats-0.1.4 → compstats-0.1.6}/CompStats/bootstrap.py +0 -0
  12. {compstats-0.1.4 → compstats-0.1.6}/CompStats/measurements.py +0 -0
  13. {compstats-0.1.4 → compstats-0.1.6}/CompStats/performance.py +0 -0
  14. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/__init__.py +0 -0
  15. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/data.csv +0 -0
  16. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_bootstrap.py +0 -0
  17. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_measurements.py +0 -0
  18. {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_performance.py +0 -0
  19. {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/SOURCES.txt +0 -0
  20. {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/dependency_links.txt +0 -0
  21. {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/requires.txt +0 -0
  22. {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/top_level.txt +0 -0
  23. {compstats-0.1.4 → compstats-0.1.6}/LICENSE +0 -0
  24. {compstats-0.1.4 → compstats-0.1.6}/MANIFEST.in +0 -0
  25. {compstats-0.1.4 → compstats-0.1.6}/pyproject.toml +0 -0
  26. {compstats-0.1.4 → compstats-0.1.6}/setup.cfg +0 -0
  27. {compstats-0.1.4 → compstats-0.1.6}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.4'
14
+ __version__ = '0.1.6'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -107,6 +107,7 @@ class Perf(object):
107
107
  self.num_samples = num_samples
108
108
  self.n_jobs = n_jobs
109
109
  self.use_tqdm = use_tqdm
110
+ self.sorting_func = np.linalg.norm
110
111
  self._init()
111
112
 
112
113
  def _init(self):
@@ -139,20 +140,41 @@ class Perf(object):
139
140
  ins = klass(**params)
140
141
  ins.predictions = dict(self.predictions)
141
142
  ins._statistic_samples._samples = self.statistic_samples._samples
143
+ ins.sorting_func = self.sorting_func
142
144
  return ins
143
145
 
144
146
  def __repr__(self):
145
147
  """Prediction statistics with standard error in parenthesis"""
146
- return f"<{self.__class__.__name__}>\n{self}"
147
-
148
+ arg = 'score_func' if self.error_func is None else 'error_func'
149
+ func_name = self.statistic_func.__name__
150
+ statistic = self.statistic
151
+ if isinstance(statistic, dict):
152
+ return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
153
+ elif isinstance(statistic, float):
154
+ return f"<{self.__class__.__name__}({arg}={func_name}, statistic={statistic:0.4f}, se={self.se:0.4f})>"
155
+ desc = [f'{k:0.4f}' for k in statistic]
156
+ desc = ', '.join(desc)
157
+ desc_se = [f'{k:0.4f}' for k in self.se]
158
+ desc_se = ', '.join(desc_se)
159
+ return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
160
+
148
161
  def __str__(self):
149
162
  """Prediction statistics with standard error in parenthesis"""
163
+ if not isinstance(self.statistic, dict):
164
+ return self.__repr__()
150
165
 
151
166
  se = self.se
152
167
  output = ["Statistic with its standard error (se)"]
153
168
  output.append("statistic (se)")
154
169
  for key, value in self.statistic.items():
155
- output.append(f'{value:0.4f} ({se[key]:0.4f}) <= {key}')
170
+ if isinstance(value, float):
171
+ desc = f'{value:0.4f} ({se[key]:0.4f}) <= {key}'
172
+ else:
173
+ desc = [f'{v:0.4f} ({k:0.4f})'
174
+ for v, k in zip(value, se[key])]
175
+ desc = ', '.join(desc)
176
+ desc = f'{desc} <= {key}'
177
+ output.append(desc)
156
178
  return "\n".join(output)
157
179
 
158
180
  def __call__(self, y_pred, name=None):
@@ -162,6 +184,7 @@ class Perf(object):
162
184
  if k == 0:
163
185
  k = 1
164
186
  name = f'alg-{k}'
187
+ self.best = None
165
188
  self.predictions[name] = np.asanyarray(y_pred)
166
189
  samples = self._statistic_samples
167
190
  calls = samples.calls
@@ -169,7 +192,7 @@ class Perf(object):
169
192
  del calls[name]
170
193
  return self
171
194
 
172
- def difference(self, wrt_to: str=None):
195
+ def difference(self, wrt: str=None):
173
196
  """Compute the difference w.r.t any algorithm by default is the best
174
197
 
175
198
  >>> from sklearn.svm import LinearSVC
@@ -190,34 +213,61 @@ class Perf(object):
190
213
  difference p-values w.r.t alg-1
191
214
  forest 0.06
192
215
  """
193
- if wrt_to is None:
194
- wrt_to = self.best[0]
195
- base = self.statistic_samples.calls[wrt_to]
216
+ if wrt is None:
217
+ wrt = self.best
218
+ if isinstance(wrt, str):
219
+ base = self.statistic_samples.calls[wrt]
220
+ else:
221
+ base = np.array([self.statistic_samples.calls[key][:, col]
222
+ for col, key in enumerate(wrt)]).T
196
223
  sign = 1 if self.statistic_samples.BiB else -1
197
224
  diff = dict()
198
225
  for k, v in self.statistic_samples.calls.items():
199
- if k == wrt_to:
226
+ if base.ndim == 1 and k == wrt:
200
227
  continue
201
228
  diff[k] = sign * (base - v)
202
229
  diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
203
- statistic=self.statistic,
204
- best=self.best[0])
230
+ statistic=self.statistic)
231
+ diff_ins.sorting_func = self.sorting_func
205
232
  diff_ins.statistic_samples.calls = diff
206
- diff_ins.statistic_samples.info['best'] = self.best[0]
233
+ diff_ins.statistic_samples.info['best'] = self.best
234
+ diff_ins.best = self.best
207
235
  return diff_ins
208
236
 
209
237
  @property
210
238
  def best(self):
211
- """Best system"""
212
-
213
- try:
239
+ """System with best performance"""
240
+ if hasattr(self, '_best') and self._best is not None:
214
241
  return self._best
215
- except AttributeError:
216
- statistic = [(k, v) for k, v in self.statistic.items()]
217
- statistic = sorted(statistic, key=lambda x: x[1],
218
- reverse=self.statistic_samples.BiB)
219
- self._best = statistic[0]
242
+ BiB = True if self.statistic_samples.BiB else False
243
+ keys = np.array(list(self.statistic.keys()))
244
+ data = np.asanyarray([self.statistic[k]
245
+ for k in keys])
246
+ if isinstance(self.statistic[keys[0]], np.ndarray):
247
+ if BiB:
248
+ best = data.argmax(axis=0)
249
+ else:
250
+ best = data.argmin(axis=0)
251
+ else:
252
+ if BiB:
253
+ best = data.argmax()
254
+ else:
255
+ best = data.argmin()
256
+ self._best = keys[best]
220
257
  return self._best
258
+
259
+ @best.setter
260
+ def best(self, value):
261
+ self._best = value
262
+
263
+ @property
264
+ def sorting_func(self):
265
+ """Rank systems when multiple performances are used"""
266
+ return self._sorting_func
267
+
268
+ @sorting_func.setter
269
+ def sorting_func(self, value):
270
+ self._sorting_func = value
221
271
 
222
272
  @property
223
273
  def statistic(self):
@@ -241,7 +291,10 @@ class Perf(object):
241
291
 
242
292
  data = sorted([(k, self.statistic_func(self.y_true, v))
243
293
  for k, v in self.predictions.items()],
244
- key=lambda x: x[1], reverse=self.statistic_samples.BiB)
294
+ key=lambda x: self.sorting_func(x[1]),
295
+ reverse=self.statistic_samples.BiB)
296
+ if len(data) == 1:
297
+ return data[0][1]
245
298
  return dict(data)
246
299
 
247
300
  @property
@@ -264,7 +317,10 @@ class Perf(object):
264
317
  {'alg-1': 0.0, 'forest': 0.026945730782184187}
265
318
  """
266
319
 
267
- return SE(self.statistic_samples)
320
+ output = SE(self.statistic_samples)
321
+ if len(output) == 1:
322
+ return list(output.values())[0]
323
+ return output
268
324
 
269
325
  def plot(self, **kwargs):
270
326
  """plot with seaborn
@@ -416,22 +472,59 @@ class Difference:
416
472
  """
417
473
 
418
474
  statistic_samples:StatisticSamples=None
419
- best:str=None
420
475
  statistic:dict=None
476
+ best:str=None
477
+
478
+ @property
479
+ def sorting_func(self):
480
+ """Rank systems when multiple performances are used"""
481
+ return self._sorting_func
482
+
483
+ @sorting_func.setter
484
+ def sorting_func(self, value):
485
+ self._sorting_func = value
421
486
 
422
487
  def __repr__(self):
423
488
  """p-value"""
424
- return f"<{self.__class__.__name__}>\n{self}"
489
+ return f"<{self.__class__.__name__}>\n{self}"
425
490
 
426
491
  def __str__(self):
427
492
  """p-value"""
428
- output = [f"difference p-values w.r.t {self.best}"]
429
- for k, v in self.p_value().items():
430
- output.append(f'{v:0.4f} <= {k}')
493
+ if isinstance(self.best, str):
494
+ best = f' w.r.t {self.best}'
495
+ else:
496
+ best = ''
497
+ output = [f"difference p-values {best}"]
498
+ best = self.best
499
+ if isinstance(best, np.ndarray):
500
+ desc = ', '.join(best)
501
+ output.append(f'{desc} <= Best')
502
+ for key, value in self.p_value().items():
503
+ if isinstance(value, float):
504
+ output.append(f'{value:0.4f} <= {key}')
505
+ else:
506
+ desc = [f'{v:0.4f}' for v in value]
507
+ desc = ', '.join(desc)
508
+ desc = f'{desc} <= {key}'
509
+ output.append(desc)
431
510
  return "\n".join(output)
432
511
 
433
- def p_value(self):
512
+ def _delta_best(self):
513
+ """Compute multiple delta"""
514
+ if isinstance(self.best, str):
515
+ return self.statistic[self.best]
516
+ keys = np.unique(self.best)
517
+ statistic = np.array([self.statistic[k]
518
+ for k in keys])
519
+ m = {v: k for k, v in enumerate(keys)}
520
+ best = np.array([m[x] for x in self.best])
521
+ return statistic[best, np.arange(best.shape[0])]
522
+
523
+ def p_value(self, right:bool=True):
434
524
  """Compute p_value of the differences
525
+
526
+ :param right: Estimate the p-value using :math:`\\text{sample} \\geq 2\\delta`
527
+ :type right: bool
435
528
 
436
529
  >>> from sklearn.svm import LinearSVC
437
530
  >>> from sklearn.ensemble import RandomForestClassifier
@@ -452,10 +545,20 @@ class Difference:
452
545
  """
453
546
  values = []
454
547
  sign = 1 if self.statistic_samples.BiB else -1
548
+ delta_best = self._delta_best()
455
549
  for k, v in self.statistic_samples.calls.items():
456
- delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
457
- values.append((k, (v > delta).mean()))
458
- values.sort(key=lambda x: x[1])
550
+ delta = 2 * sign * (delta_best - self.statistic[k])
551
+ if not isinstance(delta_best, np.ndarray):
552
+ if right:
553
+ values.append((k, (v >= delta).mean()))
554
+ else:
555
+ values.append((k, (v <= 0).mean()))
556
+ else:
557
+ if right:
558
+ values.append((k, (v >= delta).mean(axis=0)))
559
+ else:
560
+ values.append((k, (v <= 0).mean(axis=0)))
561
+ values.sort(key=lambda x: self.sorting_func(x[1]))
459
562
  return dict(values)
460
563
 
461
564
  def plot(self, **kwargs):