CompStats 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/__init__.py +1 -1
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/interface.py +133 -30
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/metrics.py +75 -29
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_interface.py +99 -15
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_metrics.py +34 -57
- compstats-0.1.6/CompStats/utils.py +57 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/PKG-INFO +20 -17
- {compstats-0.1.4 → compstats-0.1.6}/PKG-INFO +20 -17
- {compstats-0.1.4 → compstats-0.1.6}/README.rst +19 -16
- compstats-0.1.4/CompStats/utils.py +0 -55
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/measurements.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/performance.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/LICENSE +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/MANIFEST.in +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/pyproject.toml +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/setup.cfg +0 -0
- {compstats-0.1.4 → compstats-0.1.6}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.6'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -107,6 +107,7 @@ class Perf(object):
|
|
|
107
107
|
self.num_samples = num_samples
|
|
108
108
|
self.n_jobs = n_jobs
|
|
109
109
|
self.use_tqdm = use_tqdm
|
|
110
|
+
self.sorting_func = np.linalg.norm
|
|
110
111
|
self._init()
|
|
111
112
|
|
|
112
113
|
def _init(self):
|
|
@@ -139,20 +140,41 @@ class Perf(object):
|
|
|
139
140
|
ins = klass(**params)
|
|
140
141
|
ins.predictions = dict(self.predictions)
|
|
141
142
|
ins._statistic_samples._samples = self.statistic_samples._samples
|
|
143
|
+
ins.sorting_func = self.sorting_func
|
|
142
144
|
return ins
|
|
143
145
|
|
|
144
146
|
def __repr__(self):
|
|
145
147
|
"""Prediction statistics with standard error in parenthesis"""
|
|
146
|
-
|
|
147
|
-
|
|
148
|
+
arg = 'score_func' if self.error_func is None else 'error_func'
|
|
149
|
+
func_name = self.statistic_func.__name__
|
|
150
|
+
statistic = self.statistic
|
|
151
|
+
if isinstance(statistic, dict):
|
|
152
|
+
return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
|
|
153
|
+
elif isinstance(statistic, float):
|
|
154
|
+
return f"<{self.__class__.__name__}({arg}={func_name}, statistic={statistic:0.4f}, se={self.se:0.4f})>"
|
|
155
|
+
desc = [f'{k:0.4f}' for k in statistic]
|
|
156
|
+
desc = ', '.join(desc)
|
|
157
|
+
desc_se = [f'{k:0.4f}' for k in self.se]
|
|
158
|
+
desc_se = ', '.join(desc_se)
|
|
159
|
+
return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
|
|
160
|
+
|
|
148
161
|
def __str__(self):
|
|
149
162
|
"""Prediction statistics with standard error in parenthesis"""
|
|
163
|
+
if not isinstance(self.statistic, dict):
|
|
164
|
+
return self.__repr__()
|
|
150
165
|
|
|
151
166
|
se = self.se
|
|
152
167
|
output = ["Statistic with its standard error (se)"]
|
|
153
168
|
output.append("statistic (se)")
|
|
154
169
|
for key, value in self.statistic.items():
|
|
155
|
-
|
|
170
|
+
if isinstance(value, float):
|
|
171
|
+
desc = f'{value:0.4f} ({se[key]:0.4f}) <= {key}'
|
|
172
|
+
else:
|
|
173
|
+
desc = [f'{v:0.4f} ({k:0.4f})'
|
|
174
|
+
for v, k in zip(value, se[key])]
|
|
175
|
+
desc = ', '.join(desc)
|
|
176
|
+
desc = f'{desc} <= {key}'
|
|
177
|
+
output.append(desc)
|
|
156
178
|
return "\n".join(output)
|
|
157
179
|
|
|
158
180
|
def __call__(self, y_pred, name=None):
|
|
@@ -162,6 +184,7 @@ class Perf(object):
|
|
|
162
184
|
if k == 0:
|
|
163
185
|
k = 1
|
|
164
186
|
name = f'alg-{k}'
|
|
187
|
+
self.best = None
|
|
165
188
|
self.predictions[name] = np.asanyarray(y_pred)
|
|
166
189
|
samples = self._statistic_samples
|
|
167
190
|
calls = samples.calls
|
|
@@ -169,7 +192,7 @@ class Perf(object):
|
|
|
169
192
|
del calls[name]
|
|
170
193
|
return self
|
|
171
194
|
|
|
172
|
-
def difference(self,
|
|
195
|
+
def difference(self, wrt: str=None):
|
|
173
196
|
"""Compute the difference w.r.t any algorithm by default is the best
|
|
174
197
|
|
|
175
198
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -190,34 +213,61 @@ class Perf(object):
|
|
|
190
213
|
difference p-values w.r.t alg-1
|
|
191
214
|
forest 0.06
|
|
192
215
|
"""
|
|
193
|
-
if
|
|
194
|
-
|
|
195
|
-
|
|
216
|
+
if wrt is None:
|
|
217
|
+
wrt = self.best
|
|
218
|
+
if isinstance(wrt, str):
|
|
219
|
+
base = self.statistic_samples.calls[wrt]
|
|
220
|
+
else:
|
|
221
|
+
base = np.array([self.statistic_samples.calls[key][:, col]
|
|
222
|
+
for col, key in enumerate(wrt)]).T
|
|
196
223
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
197
224
|
diff = dict()
|
|
198
225
|
for k, v in self.statistic_samples.calls.items():
|
|
199
|
-
if k ==
|
|
226
|
+
if base.ndim == 1 and k == wrt:
|
|
200
227
|
continue
|
|
201
228
|
diff[k] = sign * (base - v)
|
|
202
229
|
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
203
|
-
statistic=self.statistic
|
|
204
|
-
|
|
230
|
+
statistic=self.statistic)
|
|
231
|
+
diff_ins.sorting_func = self.sorting_func
|
|
205
232
|
diff_ins.statistic_samples.calls = diff
|
|
206
|
-
diff_ins.statistic_samples.info['best'] = self.best
|
|
233
|
+
diff_ins.statistic_samples.info['best'] = self.best
|
|
234
|
+
diff_ins.best = self.best
|
|
207
235
|
return diff_ins
|
|
208
236
|
|
|
209
237
|
@property
|
|
210
238
|
def best(self):
|
|
211
|
-
"""
|
|
212
|
-
|
|
213
|
-
try:
|
|
239
|
+
"""System with best performance"""
|
|
240
|
+
if hasattr(self, '_best') and self._best is not None:
|
|
214
241
|
return self._best
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
242
|
+
BiB = True if self.statistic_samples.BiB else False
|
|
243
|
+
keys = np.array(list(self.statistic.keys()))
|
|
244
|
+
data = np.asanyarray([self.statistic[k]
|
|
245
|
+
for k in keys])
|
|
246
|
+
if isinstance(self.statistic[keys[0]], np.ndarray):
|
|
247
|
+
if BiB:
|
|
248
|
+
best = data.argmax(axis=0)
|
|
249
|
+
else:
|
|
250
|
+
best = data.argmin(axis=0)
|
|
251
|
+
else:
|
|
252
|
+
if BiB:
|
|
253
|
+
best = data.argmax()
|
|
254
|
+
else:
|
|
255
|
+
best = data.argmin()
|
|
256
|
+
self._best = keys[best]
|
|
220
257
|
return self._best
|
|
258
|
+
|
|
259
|
+
@best.setter
|
|
260
|
+
def best(self, value):
|
|
261
|
+
self._best = value
|
|
262
|
+
|
|
263
|
+
@property
|
|
264
|
+
def sorting_func(self):
|
|
265
|
+
"""Rank systems when multiple performances are used"""
|
|
266
|
+
return self._sorting_func
|
|
267
|
+
|
|
268
|
+
@sorting_func.setter
|
|
269
|
+
def sorting_func(self, value):
|
|
270
|
+
self._sorting_func = value
|
|
221
271
|
|
|
222
272
|
@property
|
|
223
273
|
def statistic(self):
|
|
@@ -241,7 +291,10 @@ class Perf(object):
|
|
|
241
291
|
|
|
242
292
|
data = sorted([(k, self.statistic_func(self.y_true, v))
|
|
243
293
|
for k, v in self.predictions.items()],
|
|
244
|
-
key=lambda x: x[1],
|
|
294
|
+
key=lambda x: self.sorting_func(x[1]),
|
|
295
|
+
reverse=self.statistic_samples.BiB)
|
|
296
|
+
if len(data) == 1:
|
|
297
|
+
return data[0][1]
|
|
245
298
|
return dict(data)
|
|
246
299
|
|
|
247
300
|
@property
|
|
@@ -264,7 +317,10 @@ class Perf(object):
|
|
|
264
317
|
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
265
318
|
"""
|
|
266
319
|
|
|
267
|
-
|
|
320
|
+
output = SE(self.statistic_samples)
|
|
321
|
+
if len(output) == 1:
|
|
322
|
+
return list(output.values())[0]
|
|
323
|
+
return output
|
|
268
324
|
|
|
269
325
|
def plot(self, **kwargs):
|
|
270
326
|
"""plot with seaborn
|
|
@@ -416,22 +472,59 @@ class Difference:
|
|
|
416
472
|
"""
|
|
417
473
|
|
|
418
474
|
statistic_samples:StatisticSamples=None
|
|
419
|
-
best:str=None
|
|
420
475
|
statistic:dict=None
|
|
476
|
+
best:str=None
|
|
477
|
+
|
|
478
|
+
@property
|
|
479
|
+
def sorting_func(self):
|
|
480
|
+
"""Rank systems when multiple performances are used"""
|
|
481
|
+
return self._sorting_func
|
|
482
|
+
|
|
483
|
+
@sorting_func.setter
|
|
484
|
+
def sorting_func(self, value):
|
|
485
|
+
self._sorting_func = value
|
|
421
486
|
|
|
422
487
|
def __repr__(self):
|
|
423
488
|
"""p-value"""
|
|
424
|
-
return f"<{self.__class__.__name__}>\n{self}"
|
|
489
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
425
490
|
|
|
426
491
|
def __str__(self):
|
|
427
492
|
"""p-value"""
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
493
|
+
if isinstance(self.best, str):
|
|
494
|
+
best = f' w.r.t {self.best}'
|
|
495
|
+
else:
|
|
496
|
+
best = ''
|
|
497
|
+
output = [f"difference p-values {best}"]
|
|
498
|
+
best = self.best
|
|
499
|
+
if isinstance(best, np.ndarray):
|
|
500
|
+
desc = ', '.join(best)
|
|
501
|
+
output.append(f'{desc} <= Best')
|
|
502
|
+
for key, value in self.p_value().items():
|
|
503
|
+
if isinstance(value, float):
|
|
504
|
+
output.append(f'{value:0.4f} <= {key}')
|
|
505
|
+
else:
|
|
506
|
+
desc = [f'{v:0.4f}' for v in value]
|
|
507
|
+
desc = ', '.join(desc)
|
|
508
|
+
desc = f'{desc} <= {key}'
|
|
509
|
+
output.append(desc)
|
|
431
510
|
return "\n".join(output)
|
|
432
511
|
|
|
433
|
-
def
|
|
512
|
+
def _delta_best(self):
|
|
513
|
+
"""Compute multiple delta"""
|
|
514
|
+
if isinstance(self.best, str):
|
|
515
|
+
return self.statistic[self.best]
|
|
516
|
+
keys = np.unique(self.best)
|
|
517
|
+
statistic = np.array([self.statistic[k]
|
|
518
|
+
for k in keys])
|
|
519
|
+
m = {v: k for k, v in enumerate(keys)}
|
|
520
|
+
best = np.array([m[x] for x in self.best])
|
|
521
|
+
return statistic[best, np.arange(best.shape[0])]
|
|
522
|
+
|
|
523
|
+
def p_value(self, right:bool=True):
|
|
434
524
|
"""Compute p_value of the differences
|
|
525
|
+
|
|
526
|
+
:param right: Estimate the p-value using :math:`\\text{sample} \\geq 2\\delta`
|
|
527
|
+
:type right: bool
|
|
435
528
|
|
|
436
529
|
>>> from sklearn.svm import LinearSVC
|
|
437
530
|
>>> from sklearn.ensemble import RandomForestClassifier
|
|
@@ -452,10 +545,20 @@ class Difference:
|
|
|
452
545
|
"""
|
|
453
546
|
values = []
|
|
454
547
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
548
|
+
delta_best = self._delta_best()
|
|
455
549
|
for k, v in self.statistic_samples.calls.items():
|
|
456
|
-
delta = 2 * sign * (
|
|
457
|
-
|
|
458
|
-
|
|
550
|
+
delta = 2 * sign * (delta_best - self.statistic[k])
|
|
551
|
+
if not isinstance(delta_best, np.ndarray):
|
|
552
|
+
if right:
|
|
553
|
+
values.append((k, (v >= delta).mean()))
|
|
554
|
+
else:
|
|
555
|
+
values.append((k, (v <= 0).mean()))
|
|
556
|
+
else:
|
|
557
|
+
if right:
|
|
558
|
+
values.append((k, (v >= delta).mean(axis=0)))
|
|
559
|
+
else:
|
|
560
|
+
values.append((k, (v <= 0).mean(axis=0)))
|
|
561
|
+
values.sort(key=lambda x: self.sorting_func(x[1]))
|
|
459
562
|
return dict(values)
|
|
460
563
|
|
|
461
564
|
def plot(self, **kwargs):
|