CompStats 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/__init__.py +1 -1
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/interface.py +90 -33
- compstats-0.1.5/CompStats/metrics.py +599 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/performance.py +29 -20
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_interface.py +2 -2
- compstats-0.1.5/CompStats/tests/test_metrics.py +477 -0
- compstats-0.1.5/CompStats/utils.py +57 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/PKG-INFO +67 -2
- {compstats-0.1.3 → compstats-0.1.5}/PKG-INFO +67 -2
- compstats-0.1.5/README.rst +93 -0
- compstats-0.1.3/CompStats/metrics.py +0 -197
- compstats-0.1.3/CompStats/tests/test_metrics.py +0 -56
- compstats-0.1.3/CompStats/utils.py +0 -25
- compstats-0.1.3/README.rst +0 -28
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/measurements.py +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/LICENSE +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/MANIFEST.in +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/pyproject.toml +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/setup.cfg +0 -0
- {compstats-0.1.3 → compstats-0.1.5}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.5'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -31,8 +31,8 @@ class Perf(object):
|
|
|
31
31
|
:type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
32
32
|
:param error_func: Function to measure the performance where the best algorithm has the lowest value.
|
|
33
33
|
:type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
34
|
-
:param
|
|
35
|
-
:type
|
|
34
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
35
|
+
:type y_pred: numpy.ndarray
|
|
36
36
|
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
37
37
|
:type kwargs: numpy.ndarray
|
|
38
38
|
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
@@ -59,19 +59,21 @@ class Perf(object):
|
|
|
59
59
|
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
60
60
|
>>> perf
|
|
61
61
|
<Perf>
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
Statistic with its standard error (se)
|
|
63
|
+
statistic (se)
|
|
64
|
+
0.9792 (0.0221) <= alg-1
|
|
65
|
+
0.9744 (0.0246) <= forest
|
|
65
66
|
|
|
66
67
|
If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
|
|
67
68
|
|
|
68
69
|
>>> lr = LogisticRegression().fit(X_train, y_train)
|
|
69
70
|
>>> perf(lr.predict(X_val), name='Log. Reg.')
|
|
70
71
|
<Perf>
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
Statistic with its standard error (se)
|
|
73
|
+
statistic (se)
|
|
74
|
+
1.0000 (0.0000) <= Log. Reg.
|
|
75
|
+
0.9792 (0.0221) <= alg-1
|
|
76
|
+
0.9744 (0.0246) <= forest
|
|
75
77
|
|
|
76
78
|
The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
|
|
77
79
|
|
|
@@ -79,13 +81,14 @@ class Perf(object):
|
|
|
79
81
|
>>> perf_error.error_func = lambda y, hy: (y != hy).mean()
|
|
80
82
|
>>> perf_error
|
|
81
83
|
<Perf>
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
Statistic with its standard error (se)
|
|
85
|
+
statistic (se)
|
|
86
|
+
0.0000 (0.0000) <= Log. Reg.
|
|
87
|
+
0.0222 (0.0237) <= alg-1
|
|
88
|
+
0.0222 (0.0215) <= forest
|
|
86
89
|
|
|
87
90
|
"""
|
|
88
|
-
def __init__(self, y_true, *
|
|
91
|
+
def __init__(self, y_true, *y_pred,
|
|
89
92
|
score_func=balanced_accuracy_score,
|
|
90
93
|
error_func=None,
|
|
91
94
|
num_samples: int=500,
|
|
@@ -96,7 +99,7 @@ class Perf(object):
|
|
|
96
99
|
self.score_func = score_func
|
|
97
100
|
self.error_func = error_func
|
|
98
101
|
algs = {}
|
|
99
|
-
for k, v in enumerate(
|
|
102
|
+
for k, v in enumerate(y_pred):
|
|
100
103
|
algs[f'alg-{k+1}'] = np.asanyarray(v)
|
|
101
104
|
algs.update(**kwargs)
|
|
102
105
|
self.predictions = algs
|
|
@@ -104,6 +107,7 @@ class Perf(object):
|
|
|
104
107
|
self.num_samples = num_samples
|
|
105
108
|
self.n_jobs = n_jobs
|
|
106
109
|
self.use_tqdm = use_tqdm
|
|
110
|
+
self.sorting_func = np.linalg.norm
|
|
107
111
|
self._init()
|
|
108
112
|
|
|
109
113
|
def _init(self):
|
|
@@ -136,19 +140,30 @@ class Perf(object):
|
|
|
136
140
|
ins = klass(**params)
|
|
137
141
|
ins.predictions = dict(self.predictions)
|
|
138
142
|
ins._statistic_samples._samples = self.statistic_samples._samples
|
|
143
|
+
ins.sorting_func = self.sorting_func
|
|
139
144
|
return ins
|
|
140
|
-
|
|
145
|
+
|
|
141
146
|
def __repr__(self):
|
|
142
147
|
"""Prediction statistics with standard error in parenthesis"""
|
|
143
|
-
|
|
148
|
+
arg = 'score_func' if self.error_func is None else 'error_func'
|
|
149
|
+
func_name = self.statistic_func.__name__
|
|
150
|
+
return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
|
|
144
151
|
|
|
145
152
|
def __str__(self):
|
|
146
153
|
"""Prediction statistics with standard error in parenthesis"""
|
|
147
154
|
|
|
148
|
-
se = self.se
|
|
149
|
-
output = ["
|
|
150
|
-
|
|
151
|
-
|
|
155
|
+
se = self.se
|
|
156
|
+
output = ["Statistic with its standard error (se)"]
|
|
157
|
+
output.append("statistic (se)")
|
|
158
|
+
for key, value in self.statistic.items():
|
|
159
|
+
if isinstance(value, float):
|
|
160
|
+
desc = f'{value:0.4f} ({se[key]:0.4f}) <= {key}'
|
|
161
|
+
else:
|
|
162
|
+
desc = [f'{v:0.4f} ({k:0.4f})'
|
|
163
|
+
for v, k in zip(value, se[key])]
|
|
164
|
+
desc = ', '.join(desc)
|
|
165
|
+
desc = f'{desc} <= {key}'
|
|
166
|
+
output.append(desc)
|
|
152
167
|
return "\n".join(output)
|
|
153
168
|
|
|
154
169
|
def __call__(self, y_pred, name=None):
|
|
@@ -196,8 +211,9 @@ class Perf(object):
|
|
|
196
211
|
continue
|
|
197
212
|
diff[k] = sign * (base - v)
|
|
198
213
|
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
199
|
-
statistic=self.statistic
|
|
214
|
+
statistic=self.statistic,
|
|
200
215
|
best=self.best[0])
|
|
216
|
+
diff_ins.sorting_func = self.sorting_func
|
|
201
217
|
diff_ins.statistic_samples.calls = diff
|
|
202
218
|
diff_ins.statistic_samples.info['best'] = self.best[0]
|
|
203
219
|
return diff_ins
|
|
@@ -209,12 +225,23 @@ class Perf(object):
|
|
|
209
225
|
try:
|
|
210
226
|
return self._best
|
|
211
227
|
except AttributeError:
|
|
212
|
-
statistic = [(k, v) for k, v in self.statistic
|
|
213
|
-
statistic = sorted(statistic,
|
|
228
|
+
statistic = [(k, v) for k, v in self.statistic.items()]
|
|
229
|
+
statistic = sorted(statistic,
|
|
230
|
+
key=lambda x: self.sorting_func(x[1]),
|
|
214
231
|
reverse=self.statistic_samples.BiB)
|
|
215
232
|
self._best = statistic[0]
|
|
216
233
|
return self._best
|
|
234
|
+
|
|
235
|
+
@property
|
|
236
|
+
def sorting_func(self):
|
|
237
|
+
"""Rank systems when multiple performances are used"""
|
|
238
|
+
return self._sorting_func
|
|
239
|
+
|
|
240
|
+
@sorting_func.setter
|
|
241
|
+
def sorting_func(self, value):
|
|
242
|
+
self._sorting_func = value
|
|
217
243
|
|
|
244
|
+
@property
|
|
218
245
|
def statistic(self):
|
|
219
246
|
"""Statistic
|
|
220
247
|
|
|
@@ -230,15 +257,17 @@ class Perf(object):
|
|
|
230
257
|
>>> hy = m.predict(X_val)
|
|
231
258
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
232
259
|
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
233
|
-
>>> perf.statistic
|
|
260
|
+
>>> perf.statistic
|
|
234
261
|
{'alg-1': 1.0, 'forest': 0.9500891265597148}
|
|
235
262
|
"""
|
|
236
263
|
|
|
237
264
|
data = sorted([(k, self.statistic_func(self.y_true, v))
|
|
238
265
|
for k, v in self.predictions.items()],
|
|
239
|
-
key=lambda x: x[1],
|
|
266
|
+
key=lambda x: self.sorting_func(x[1]),
|
|
267
|
+
reverse=self.statistic_samples.BiB)
|
|
240
268
|
return dict(data)
|
|
241
269
|
|
|
270
|
+
@property
|
|
242
271
|
def se(self):
|
|
243
272
|
"""Standard Error
|
|
244
273
|
|
|
@@ -254,9 +283,10 @@ class Perf(object):
|
|
|
254
283
|
>>> hy = m.predict(X_val)
|
|
255
284
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
256
285
|
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
257
|
-
>>>
|
|
286
|
+
>>> perf.se
|
|
258
287
|
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
259
288
|
"""
|
|
289
|
+
|
|
260
290
|
return SE(self.statistic_samples)
|
|
261
291
|
|
|
262
292
|
def plot(self, **kwargs):
|
|
@@ -405,13 +435,22 @@ class Difference:
|
|
|
405
435
|
>>> diff
|
|
406
436
|
<Difference>
|
|
407
437
|
difference p-values w.r.t alg-1
|
|
408
|
-
|
|
438
|
+
0.0780 <= forest
|
|
409
439
|
"""
|
|
410
440
|
|
|
411
441
|
statistic_samples:StatisticSamples=None
|
|
412
442
|
best:str=None
|
|
413
443
|
statistic:dict=None
|
|
414
444
|
|
|
445
|
+
@property
|
|
446
|
+
def sorting_func(self):
|
|
447
|
+
"""Rank systems when multiple performances are used"""
|
|
448
|
+
return self._sorting_func
|
|
449
|
+
|
|
450
|
+
@sorting_func.setter
|
|
451
|
+
def sorting_func(self, value):
|
|
452
|
+
self._sorting_func = value
|
|
453
|
+
|
|
415
454
|
def __repr__(self):
|
|
416
455
|
"""p-value"""
|
|
417
456
|
return f"<{self.__class__.__name__}>\n{self}"
|
|
@@ -419,12 +458,20 @@ class Difference:
|
|
|
419
458
|
def __str__(self):
|
|
420
459
|
"""p-value"""
|
|
421
460
|
output = [f"difference p-values w.r.t {self.best}"]
|
|
422
|
-
for
|
|
423
|
-
|
|
461
|
+
for key, value in self.p_value().items():
|
|
462
|
+
if isinstance(value, float):
|
|
463
|
+
output.append(f'{value:0.4f} <= {key}')
|
|
464
|
+
else:
|
|
465
|
+
desc = [f'{v:0.4f}' for v in value]
|
|
466
|
+
desc = ', '.join(desc)
|
|
467
|
+
desc = f'{desc} <= {key}'
|
|
424
468
|
return "\n".join(output)
|
|
425
469
|
|
|
426
|
-
def p_value(self):
|
|
470
|
+
def p_value(self, right:bool=True):
|
|
427
471
|
"""Compute p_value of the differences
|
|
472
|
+
|
|
473
|
+
:param right: Estimate the p-value using :math:`\\text{sample} \\geq 2\\delta`
|
|
474
|
+
:type right: bool
|
|
428
475
|
|
|
429
476
|
>>> from sklearn.svm import LinearSVC
|
|
430
477
|
>>> from sklearn.ensemble import RandomForestClassifier
|
|
@@ -445,10 +492,20 @@ class Difference:
|
|
|
445
492
|
"""
|
|
446
493
|
values = []
|
|
447
494
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
495
|
+
ndim = self.statistic[self.best].ndim
|
|
448
496
|
for k, v in self.statistic_samples.calls.items():
|
|
449
497
|
delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
|
|
450
|
-
|
|
451
|
-
|
|
498
|
+
if ndim == 0:
|
|
499
|
+
if right:
|
|
500
|
+
values.append((k, (v >= delta).mean()))
|
|
501
|
+
else:
|
|
502
|
+
values.append((k, (v <= 0).mean()))
|
|
503
|
+
else:
|
|
504
|
+
if right:
|
|
505
|
+
values.append((k, (v >= delta).mean(axis=0)))
|
|
506
|
+
else:
|
|
507
|
+
values.append((k, (v <= 0).mean(axis=0)))
|
|
508
|
+
values.sort(key=lambda x: self.sorting_func(x[1]))
|
|
452
509
|
return dict(values)
|
|
453
510
|
|
|
454
511
|
def plot(self, **kwargs):
|