CompStats 0.1.4__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/__init__.py +1 -1
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/interface.py +59 -9
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/metrics.py +75 -29
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_metrics.py +24 -0
- compstats-0.1.5/CompStats/utils.py +57 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/PKG-INFO +1 -1
- {compstats-0.1.4 → compstats-0.1.5}/PKG-INFO +1 -1
- compstats-0.1.4/CompStats/utils.py +0 -55
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/measurements.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/performance.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_interface.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/LICENSE +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/MANIFEST.in +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/README.rst +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/pyproject.toml +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/setup.cfg +0 -0
- {compstats-0.1.4 → compstats-0.1.5}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.5'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -107,6 +107,7 @@ class Perf(object):
|
|
|
107
107
|
self.num_samples = num_samples
|
|
108
108
|
self.n_jobs = n_jobs
|
|
109
109
|
self.use_tqdm = use_tqdm
|
|
110
|
+
self.sorting_func = np.linalg.norm
|
|
110
111
|
self._init()
|
|
111
112
|
|
|
112
113
|
def _init(self):
|
|
@@ -139,11 +140,14 @@ class Perf(object):
|
|
|
139
140
|
ins = klass(**params)
|
|
140
141
|
ins.predictions = dict(self.predictions)
|
|
141
142
|
ins._statistic_samples._samples = self.statistic_samples._samples
|
|
143
|
+
ins.sorting_func = self.sorting_func
|
|
142
144
|
return ins
|
|
143
145
|
|
|
144
146
|
def __repr__(self):
|
|
145
147
|
"""Prediction statistics with standard error in parenthesis"""
|
|
146
|
-
|
|
148
|
+
arg = 'score_func' if self.error_func is None else 'error_func'
|
|
149
|
+
func_name = self.statistic_func.__name__
|
|
150
|
+
return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
|
|
147
151
|
|
|
148
152
|
def __str__(self):
|
|
149
153
|
"""Prediction statistics with standard error in parenthesis"""
|
|
@@ -152,7 +156,14 @@ class Perf(object):
|
|
|
152
156
|
output = ["Statistic with its standard error (se)"]
|
|
153
157
|
output.append("statistic (se)")
|
|
154
158
|
for key, value in self.statistic.items():
|
|
155
|
-
|
|
159
|
+
if isinstance(value, float):
|
|
160
|
+
desc = f'{value:0.4f} ({se[key]:0.4f}) <= {key}'
|
|
161
|
+
else:
|
|
162
|
+
desc = [f'{v:0.4f} ({k:0.4f})'
|
|
163
|
+
for v, k in zip(value, se[key])]
|
|
164
|
+
desc = ', '.join(desc)
|
|
165
|
+
desc = f'{desc} <= {key}'
|
|
166
|
+
output.append(desc)
|
|
156
167
|
return "\n".join(output)
|
|
157
168
|
|
|
158
169
|
def __call__(self, y_pred, name=None):
|
|
@@ -202,6 +213,7 @@ class Perf(object):
|
|
|
202
213
|
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
203
214
|
statistic=self.statistic,
|
|
204
215
|
best=self.best[0])
|
|
216
|
+
diff_ins.sorting_func = self.sorting_func
|
|
205
217
|
diff_ins.statistic_samples.calls = diff
|
|
206
218
|
diff_ins.statistic_samples.info['best'] = self.best[0]
|
|
207
219
|
return diff_ins
|
|
@@ -214,10 +226,20 @@ class Perf(object):
|
|
|
214
226
|
return self._best
|
|
215
227
|
except AttributeError:
|
|
216
228
|
statistic = [(k, v) for k, v in self.statistic.items()]
|
|
217
|
-
statistic = sorted(statistic,
|
|
229
|
+
statistic = sorted(statistic,
|
|
230
|
+
key=lambda x: self.sorting_func(x[1]),
|
|
218
231
|
reverse=self.statistic_samples.BiB)
|
|
219
232
|
self._best = statistic[0]
|
|
220
233
|
return self._best
|
|
234
|
+
|
|
235
|
+
@property
|
|
236
|
+
def sorting_func(self):
|
|
237
|
+
"""Rank systems when multiple performances are used"""
|
|
238
|
+
return self._sorting_func
|
|
239
|
+
|
|
240
|
+
@sorting_func.setter
|
|
241
|
+
def sorting_func(self, value):
|
|
242
|
+
self._sorting_func = value
|
|
221
243
|
|
|
222
244
|
@property
|
|
223
245
|
def statistic(self):
|
|
@@ -241,7 +263,8 @@ class Perf(object):
|
|
|
241
263
|
|
|
242
264
|
data = sorted([(k, self.statistic_func(self.y_true, v))
|
|
243
265
|
for k, v in self.predictions.items()],
|
|
244
|
-
key=lambda x: x[1],
|
|
266
|
+
key=lambda x: self.sorting_func(x[1]),
|
|
267
|
+
reverse=self.statistic_samples.BiB)
|
|
245
268
|
return dict(data)
|
|
246
269
|
|
|
247
270
|
@property
|
|
@@ -419,6 +442,15 @@ class Difference:
|
|
|
419
442
|
best:str=None
|
|
420
443
|
statistic:dict=None
|
|
421
444
|
|
|
445
|
+
@property
|
|
446
|
+
def sorting_func(self):
|
|
447
|
+
"""Rank systems when multiple performances are used"""
|
|
448
|
+
return self._sorting_func
|
|
449
|
+
|
|
450
|
+
@sorting_func.setter
|
|
451
|
+
def sorting_func(self, value):
|
|
452
|
+
self._sorting_func = value
|
|
453
|
+
|
|
422
454
|
def __repr__(self):
|
|
423
455
|
"""p-value"""
|
|
424
456
|
return f"<{self.__class__.__name__}>\n{self}"
|
|
@@ -426,12 +458,20 @@ class Difference:
|
|
|
426
458
|
def __str__(self):
|
|
427
459
|
"""p-value"""
|
|
428
460
|
output = [f"difference p-values w.r.t {self.best}"]
|
|
429
|
-
for
|
|
430
|
-
|
|
461
|
+
for key, value in self.p_value().items():
|
|
462
|
+
if isinstance(value, float):
|
|
463
|
+
output.append(f'{value:0.4f} <= {key}')
|
|
464
|
+
else:
|
|
465
|
+
desc = [f'{v:0.4f}' for v in value]
|
|
466
|
+
desc = ', '.join(desc)
|
|
467
|
+
desc = f'{desc} <= {key}'
|
|
431
468
|
return "\n".join(output)
|
|
432
469
|
|
|
433
|
-
def p_value(self):
|
|
470
|
+
def p_value(self, right:bool=True):
|
|
434
471
|
"""Compute p_value of the differences
|
|
472
|
+
|
|
473
|
+
:param right: Estimate the p-value using :math:`\\text{sample} \\geq 2\\delta`
|
|
474
|
+
:type right: bool
|
|
435
475
|
|
|
436
476
|
>>> from sklearn.svm import LinearSVC
|
|
437
477
|
>>> from sklearn.ensemble import RandomForestClassifier
|
|
@@ -452,10 +492,20 @@ class Difference:
|
|
|
452
492
|
"""
|
|
453
493
|
values = []
|
|
454
494
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
495
|
+
ndim = self.statistic[self.best].ndim
|
|
455
496
|
for k, v in self.statistic_samples.calls.items():
|
|
456
497
|
delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
|
|
457
|
-
|
|
458
|
-
|
|
498
|
+
if ndim == 0:
|
|
499
|
+
if right:
|
|
500
|
+
values.append((k, (v >= delta).mean()))
|
|
501
|
+
else:
|
|
502
|
+
values.append((k, (v <= 0).mean()))
|
|
503
|
+
else:
|
|
504
|
+
if right:
|
|
505
|
+
values.append((k, (v >= delta).mean(axis=0)))
|
|
506
|
+
else:
|
|
507
|
+
values.append((k, (v <= 0).mean(axis=0)))
|
|
508
|
+
values.sort(key=lambda x: self.sorting_func(x[1]))
|
|
459
509
|
return dict(values)
|
|
460
510
|
|
|
461
511
|
def plot(self, **kwargs):
|
|
@@ -11,15 +11,18 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
+
from functools import wraps
|
|
14
15
|
from sklearn import metrics
|
|
15
16
|
from CompStats.interface import Perf
|
|
16
|
-
from CompStats.utils import
|
|
17
|
+
from CompStats.utils import metrics_docs
|
|
18
|
+
|
|
17
19
|
|
|
18
20
|
########################################################
|
|
19
21
|
#################### Classification ####################
|
|
20
22
|
########################################################
|
|
21
23
|
|
|
22
|
-
|
|
24
|
+
|
|
25
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
23
26
|
def accuracy_score(y_true, *y_pred,
|
|
24
27
|
normalize=True, sample_weight=None,
|
|
25
28
|
num_samples: int=500,
|
|
@@ -28,6 +31,7 @@ def accuracy_score(y_true, *y_pred,
|
|
|
28
31
|
**kwargs):
|
|
29
32
|
"""accuracy_score"""
|
|
30
33
|
|
|
34
|
+
@wraps(metrics.accuracy_score)
|
|
31
35
|
def inner(y, hy):
|
|
32
36
|
return metrics.accuracy_score(y, hy,
|
|
33
37
|
normalize=normalize,
|
|
@@ -38,7 +42,7 @@ def accuracy_score(y_true, *y_pred,
|
|
|
38
42
|
**kwargs)
|
|
39
43
|
|
|
40
44
|
|
|
41
|
-
@
|
|
45
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
42
46
|
def balanced_accuracy_score(y_true, *y_pred,
|
|
43
47
|
sample_weight=None, adjusted=False,
|
|
44
48
|
num_samples: int=500,
|
|
@@ -47,6 +51,7 @@ def balanced_accuracy_score(y_true, *y_pred,
|
|
|
47
51
|
**kwargs):
|
|
48
52
|
"""balanced_accuracy_score"""
|
|
49
53
|
|
|
54
|
+
@wraps(metrics.balanced_accuracy_score)
|
|
50
55
|
def inner(y, hy):
|
|
51
56
|
return metrics.balanced_accuracy_score(y, hy,
|
|
52
57
|
adjusted=adjusted,
|
|
@@ -57,7 +62,7 @@ def balanced_accuracy_score(y_true, *y_pred,
|
|
|
57
62
|
**kwargs)
|
|
58
63
|
|
|
59
64
|
|
|
60
|
-
@
|
|
65
|
+
@metrics_docs(hy_name='y_score', attr_name='score_func')
|
|
61
66
|
def top_k_accuracy_score(y_true, *y_score, k=2,
|
|
62
67
|
normalize=True, sample_weight=None,
|
|
63
68
|
labels=None,
|
|
@@ -67,6 +72,7 @@ def top_k_accuracy_score(y_true, *y_score, k=2,
|
|
|
67
72
|
**kwargs):
|
|
68
73
|
"""top_k_accuracy_score"""
|
|
69
74
|
|
|
75
|
+
@wraps(metrics.top_k_accuracy_score)
|
|
70
76
|
def inner(y, hy):
|
|
71
77
|
return metrics.top_k_accuracy_score(y, hy, k=k,
|
|
72
78
|
normalize=normalize, sample_weight=sample_weight,
|
|
@@ -77,7 +83,7 @@ def top_k_accuracy_score(y_true, *y_score, k=2,
|
|
|
77
83
|
**kwargs)
|
|
78
84
|
|
|
79
85
|
|
|
80
|
-
@
|
|
86
|
+
@metrics_docs(hy_name='y_score', attr_name='score_func')
|
|
81
87
|
def average_precision_score(y_true, *y_score,
|
|
82
88
|
average='macro',
|
|
83
89
|
sample_weight=None,
|
|
@@ -87,6 +93,7 @@ def average_precision_score(y_true, *y_score,
|
|
|
87
93
|
**kwargs):
|
|
88
94
|
"""average_precision_score"""
|
|
89
95
|
|
|
96
|
+
@wraps(metrics.average_precision_score)
|
|
90
97
|
def inner(y, hy):
|
|
91
98
|
return metrics.average_precision_score(y, hy,
|
|
92
99
|
average=average,
|
|
@@ -97,7 +104,7 @@ def average_precision_score(y_true, *y_score,
|
|
|
97
104
|
**kwargs)
|
|
98
105
|
|
|
99
106
|
|
|
100
|
-
@
|
|
107
|
+
@metrics_docs(hy_name='y_proba', attr_name='error_func')
|
|
101
108
|
def brier_score_loss(y_true, *y_proba,
|
|
102
109
|
sample_weight=None,
|
|
103
110
|
pos_label=None,
|
|
@@ -108,6 +115,7 @@ def brier_score_loss(y_true, *y_proba,
|
|
|
108
115
|
):
|
|
109
116
|
"""brier_score_loss"""
|
|
110
117
|
|
|
118
|
+
@wraps(metrics.brier_score_loss)
|
|
111
119
|
def inner(y, hy):
|
|
112
120
|
return metrics.brier_score_loss(y, hy,
|
|
113
121
|
sample_weight=sample_weight,
|
|
@@ -116,9 +124,9 @@ def brier_score_loss(y_true, *y_proba,
|
|
|
116
124
|
num_samples=num_samples, n_jobs=n_jobs,
|
|
117
125
|
use_tqdm=use_tqdm,
|
|
118
126
|
**kwargs)
|
|
119
|
-
|
|
120
127
|
|
|
121
|
-
|
|
128
|
+
|
|
129
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
122
130
|
def f1_score(y_true, *y_pred, labels=None, pos_label=1,
|
|
123
131
|
average='binary', sample_weight=None,
|
|
124
132
|
zero_division='warn', num_samples: int=500,
|
|
@@ -126,6 +134,7 @@ def f1_score(y_true, *y_pred, labels=None, pos_label=1,
|
|
|
126
134
|
**kwargs):
|
|
127
135
|
"""f1_score"""
|
|
128
136
|
|
|
137
|
+
@wraps(metrics.f1_score)
|
|
129
138
|
def inner(y, hy):
|
|
130
139
|
return metrics.f1_score(y, hy, labels=labels,
|
|
131
140
|
pos_label=pos_label,
|
|
@@ -138,7 +147,7 @@ def f1_score(y_true, *y_pred, labels=None, pos_label=1,
|
|
|
138
147
|
**kwargs)
|
|
139
148
|
|
|
140
149
|
|
|
141
|
-
@
|
|
150
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
142
151
|
def log_loss(y_true, *y_pred,
|
|
143
152
|
normalize=True,
|
|
144
153
|
sample_weight=None,
|
|
@@ -148,6 +157,8 @@ def log_loss(y_true, *y_pred,
|
|
|
148
157
|
use_tqdm=True,
|
|
149
158
|
**kwargs):
|
|
150
159
|
"""log_loss"""
|
|
160
|
+
|
|
161
|
+
@wraps(metrics.log_loss)
|
|
151
162
|
def inner(y, hy):
|
|
152
163
|
return metrics.log_loss(y, hy, normalize=normalize,
|
|
153
164
|
sample_weight=sample_weight,
|
|
@@ -158,7 +169,7 @@ def log_loss(y_true, *y_pred,
|
|
|
158
169
|
**kwargs)
|
|
159
170
|
|
|
160
171
|
|
|
161
|
-
@
|
|
172
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
162
173
|
def precision_score(y_true,
|
|
163
174
|
*y_pred,
|
|
164
175
|
labels=None,
|
|
@@ -171,6 +182,8 @@ def precision_score(y_true,
|
|
|
171
182
|
use_tqdm=True,
|
|
172
183
|
**kwargs):
|
|
173
184
|
"""precision_score"""
|
|
185
|
+
|
|
186
|
+
@wraps(metrics.precision_score)
|
|
174
187
|
def inner(y, hy):
|
|
175
188
|
return metrics.precision_score(y, hy,
|
|
176
189
|
labels=labels,
|
|
@@ -184,7 +197,7 @@ def precision_score(y_true,
|
|
|
184
197
|
**kwargs)
|
|
185
198
|
|
|
186
199
|
|
|
187
|
-
@
|
|
200
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
188
201
|
def recall_score(y_true,
|
|
189
202
|
*y_pred,
|
|
190
203
|
labels=None,
|
|
@@ -197,6 +210,8 @@ def recall_score(y_true,
|
|
|
197
210
|
use_tqdm=True,
|
|
198
211
|
**kwargs):
|
|
199
212
|
"""recall_score"""
|
|
213
|
+
|
|
214
|
+
@wraps(metrics.recall_score)
|
|
200
215
|
def inner(y, hy):
|
|
201
216
|
return metrics.recall_score(y, hy,
|
|
202
217
|
labels=labels,
|
|
@@ -210,7 +225,7 @@ def recall_score(y_true,
|
|
|
210
225
|
**kwargs)
|
|
211
226
|
|
|
212
227
|
|
|
213
|
-
@
|
|
228
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
214
229
|
def jaccard_score(y_true,
|
|
215
230
|
*y_pred,
|
|
216
231
|
labels=None,
|
|
@@ -223,6 +238,8 @@ def jaccard_score(y_true,
|
|
|
223
238
|
use_tqdm=True,
|
|
224
239
|
**kwargs):
|
|
225
240
|
"""jaccard_score"""
|
|
241
|
+
|
|
242
|
+
@wraps(metrics.jaccard_score)
|
|
226
243
|
def inner(y, hy):
|
|
227
244
|
return metrics.jaccard_score(y, hy,
|
|
228
245
|
labels=labels,
|
|
@@ -236,7 +253,7 @@ def jaccard_score(y_true,
|
|
|
236
253
|
**kwargs)
|
|
237
254
|
|
|
238
255
|
|
|
239
|
-
@
|
|
256
|
+
@metrics_docs(hy_name='y_score', attr_name='score_func')
|
|
240
257
|
def roc_auc_score(y_true,
|
|
241
258
|
*y_score,
|
|
242
259
|
average='macro',
|
|
@@ -249,6 +266,8 @@ def roc_auc_score(y_true,
|
|
|
249
266
|
use_tqdm=True,
|
|
250
267
|
**kwargs):
|
|
251
268
|
"""roc_auc_score"""
|
|
269
|
+
|
|
270
|
+
@wraps(metrics.roc_auc_score)
|
|
252
271
|
def inner(y, hy):
|
|
253
272
|
return metrics.roc_auc_score(y, hy,
|
|
254
273
|
average=average,
|
|
@@ -262,7 +281,7 @@ def roc_auc_score(y_true,
|
|
|
262
281
|
**kwargs)
|
|
263
282
|
|
|
264
283
|
|
|
265
|
-
@
|
|
284
|
+
@metrics_docs(hy_name='y_proba', attr_name='score_func')
|
|
266
285
|
def d2_log_loss_score(y_true, *y_proba,
|
|
267
286
|
sample_weight=None,
|
|
268
287
|
labels=None,
|
|
@@ -272,6 +291,7 @@ def d2_log_loss_score(y_true, *y_proba,
|
|
|
272
291
|
**kwargs):
|
|
273
292
|
"""d2_log_loss_score"""
|
|
274
293
|
|
|
294
|
+
@wraps(metrics.d2_log_loss_score)
|
|
275
295
|
def inner(y, hy):
|
|
276
296
|
return metrics.d2_log_loss_score(y, hy,
|
|
277
297
|
sample_weight=sample_weight,
|
|
@@ -287,7 +307,7 @@ def d2_log_loss_score(y_true, *y_proba,
|
|
|
287
307
|
########################################################
|
|
288
308
|
|
|
289
309
|
|
|
290
|
-
@
|
|
310
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
291
311
|
def explained_variance_score(y_true,
|
|
292
312
|
*y_pred,
|
|
293
313
|
sample_weight=None,
|
|
@@ -298,6 +318,8 @@ def explained_variance_score(y_true,
|
|
|
298
318
|
use_tqdm=True,
|
|
299
319
|
**kwargs):
|
|
300
320
|
"""explained_variance_score"""
|
|
321
|
+
|
|
322
|
+
@wraps(metrics.explained_variance_score)
|
|
301
323
|
def inner(y, hy):
|
|
302
324
|
return metrics.explained_variance_score(y, hy,
|
|
303
325
|
sample_weight=sample_weight,
|
|
@@ -309,13 +331,15 @@ def explained_variance_score(y_true,
|
|
|
309
331
|
**kwargs)
|
|
310
332
|
|
|
311
333
|
|
|
312
|
-
@
|
|
334
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
313
335
|
def max_error(y_true, *y_pred,
|
|
314
336
|
num_samples: int=500,
|
|
315
337
|
n_jobs: int=-1,
|
|
316
338
|
use_tqdm=True,
|
|
317
339
|
**kwargs):
|
|
318
340
|
"""max_error"""
|
|
341
|
+
|
|
342
|
+
@wraps(metrics.max_error)
|
|
319
343
|
def inner(y, hy):
|
|
320
344
|
return metrics.max_error(y, hy)
|
|
321
345
|
return Perf(y_true, *y_pred, score_func=None, error_func=inner,
|
|
@@ -324,7 +348,7 @@ def max_error(y_true, *y_pred,
|
|
|
324
348
|
**kwargs)
|
|
325
349
|
|
|
326
350
|
|
|
327
|
-
@
|
|
351
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
328
352
|
def mean_absolute_error(y_true,
|
|
329
353
|
*y_pred,
|
|
330
354
|
sample_weight=None,
|
|
@@ -334,6 +358,8 @@ def mean_absolute_error(y_true,
|
|
|
334
358
|
use_tqdm=True,
|
|
335
359
|
**kwargs):
|
|
336
360
|
"""mean_absolute_error"""
|
|
361
|
+
|
|
362
|
+
@wraps(metrics.mean_absolute_error)
|
|
337
363
|
def inner(y, hy):
|
|
338
364
|
return metrics.mean_absolute_error(y, hy,
|
|
339
365
|
sample_weight=sample_weight,
|
|
@@ -345,7 +371,7 @@ def mean_absolute_error(y_true,
|
|
|
345
371
|
**kwargs)
|
|
346
372
|
|
|
347
373
|
|
|
348
|
-
@
|
|
374
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
349
375
|
def mean_squared_error(y_true,
|
|
350
376
|
*y_pred,
|
|
351
377
|
sample_weight=None,
|
|
@@ -355,6 +381,8 @@ def mean_squared_error(y_true,
|
|
|
355
381
|
use_tqdm=True,
|
|
356
382
|
**kwargs):
|
|
357
383
|
"""mean_squared_error"""
|
|
384
|
+
|
|
385
|
+
@wraps(metrics.mean_squared_error)
|
|
358
386
|
def inner(y, hy):
|
|
359
387
|
return metrics.mean_squared_error(y, hy,
|
|
360
388
|
sample_weight=sample_weight,
|
|
@@ -366,7 +394,7 @@ def mean_squared_error(y_true,
|
|
|
366
394
|
**kwargs)
|
|
367
395
|
|
|
368
396
|
|
|
369
|
-
@
|
|
397
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
370
398
|
def root_mean_squared_error(y_true,
|
|
371
399
|
*y_pred,
|
|
372
400
|
sample_weight=None,
|
|
@@ -376,6 +404,8 @@ def root_mean_squared_error(y_true,
|
|
|
376
404
|
use_tqdm=True,
|
|
377
405
|
**kwargs):
|
|
378
406
|
"""root_mean_squared_error"""
|
|
407
|
+
|
|
408
|
+
@wraps(metrics.root_mean_squared_error)
|
|
379
409
|
def inner(y, hy):
|
|
380
410
|
return metrics.root_mean_squared_error(y, hy,
|
|
381
411
|
sample_weight=sample_weight,
|
|
@@ -387,7 +417,7 @@ def root_mean_squared_error(y_true,
|
|
|
387
417
|
**kwargs)
|
|
388
418
|
|
|
389
419
|
|
|
390
|
-
@
|
|
420
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
391
421
|
def mean_squared_log_error(y_true,
|
|
392
422
|
*y_pred,
|
|
393
423
|
sample_weight=None,
|
|
@@ -397,6 +427,8 @@ def mean_squared_log_error(y_true,
|
|
|
397
427
|
use_tqdm=True,
|
|
398
428
|
**kwargs):
|
|
399
429
|
"""mean_squared_log_error"""
|
|
430
|
+
|
|
431
|
+
@wraps(metrics.mean_squared_log_error)
|
|
400
432
|
def inner(y, hy):
|
|
401
433
|
return metrics.mean_squared_log_error(y, hy,
|
|
402
434
|
sample_weight=sample_weight,
|
|
@@ -408,7 +440,7 @@ def mean_squared_log_error(y_true,
|
|
|
408
440
|
**kwargs)
|
|
409
441
|
|
|
410
442
|
|
|
411
|
-
@
|
|
443
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
412
444
|
def root_mean_squared_log_error(y_true,
|
|
413
445
|
*y_pred,
|
|
414
446
|
sample_weight=None,
|
|
@@ -418,6 +450,8 @@ def root_mean_squared_log_error(y_true,
|
|
|
418
450
|
use_tqdm=True,
|
|
419
451
|
**kwargs):
|
|
420
452
|
"""root_mean_squared_log_error"""
|
|
453
|
+
|
|
454
|
+
@wraps(metrics.root_mean_squared_log_error)
|
|
421
455
|
def inner(y, hy):
|
|
422
456
|
return metrics.root_mean_squared_log_error(y, hy,
|
|
423
457
|
sample_weight=sample_weight,
|
|
@@ -429,7 +463,7 @@ def root_mean_squared_log_error(y_true,
|
|
|
429
463
|
**kwargs)
|
|
430
464
|
|
|
431
465
|
|
|
432
|
-
@
|
|
466
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
433
467
|
def median_absolute_error(y_true,
|
|
434
468
|
*y_pred,
|
|
435
469
|
sample_weight=None,
|
|
@@ -439,10 +473,12 @@ def median_absolute_error(y_true,
|
|
|
439
473
|
use_tqdm=True,
|
|
440
474
|
**kwargs):
|
|
441
475
|
"""median_absolute_error"""
|
|
476
|
+
|
|
477
|
+
@wraps(metrics.median_absolute_error)
|
|
442
478
|
def inner(y, hy):
|
|
443
479
|
return metrics.median_absolute_error(y, hy,
|
|
444
|
-
|
|
445
|
-
|
|
480
|
+
sample_weight=sample_weight,
|
|
481
|
+
multioutput=multioutput)
|
|
446
482
|
|
|
447
483
|
return Perf(y_true, *y_pred, score_func=None, error_func=inner,
|
|
448
484
|
num_samples=num_samples, n_jobs=n_jobs,
|
|
@@ -450,7 +486,7 @@ def median_absolute_error(y_true,
|
|
|
450
486
|
**kwargs)
|
|
451
487
|
|
|
452
488
|
|
|
453
|
-
@
|
|
489
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
454
490
|
def r2_score(y_true,
|
|
455
491
|
*y_pred,
|
|
456
492
|
sample_weight=None,
|
|
@@ -461,6 +497,8 @@ def r2_score(y_true,
|
|
|
461
497
|
use_tqdm=True,
|
|
462
498
|
**kwargs):
|
|
463
499
|
"""r2_score"""
|
|
500
|
+
|
|
501
|
+
@wraps(metrics.r2_score)
|
|
464
502
|
def inner(y, hy):
|
|
465
503
|
return metrics.r2_score(y, hy,
|
|
466
504
|
sample_weight=sample_weight,
|
|
@@ -473,7 +511,7 @@ def r2_score(y_true,
|
|
|
473
511
|
**kwargs)
|
|
474
512
|
|
|
475
513
|
|
|
476
|
-
@
|
|
514
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
477
515
|
def mean_poisson_deviance(y_true,
|
|
478
516
|
*y_pred,
|
|
479
517
|
sample_weight=None,
|
|
@@ -482,6 +520,8 @@ def mean_poisson_deviance(y_true,
|
|
|
482
520
|
use_tqdm=True,
|
|
483
521
|
**kwargs):
|
|
484
522
|
"""mean_poisson_deviance"""
|
|
523
|
+
|
|
524
|
+
@wraps(metrics.mean_poisson_deviance)
|
|
485
525
|
def inner(y, hy):
|
|
486
526
|
return metrics.mean_poisson_deviance(y, hy,
|
|
487
527
|
sample_weight=sample_weight)
|
|
@@ -492,7 +532,7 @@ def mean_poisson_deviance(y_true,
|
|
|
492
532
|
**kwargs)
|
|
493
533
|
|
|
494
534
|
|
|
495
|
-
@
|
|
535
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
496
536
|
def mean_gamma_deviance(y_true,
|
|
497
537
|
*y_pred,
|
|
498
538
|
sample_weight=None,
|
|
@@ -501,6 +541,8 @@ def mean_gamma_deviance(y_true,
|
|
|
501
541
|
use_tqdm=True,
|
|
502
542
|
**kwargs):
|
|
503
543
|
"""mean_gamma_deviance"""
|
|
544
|
+
|
|
545
|
+
@wraps(metrics.mean_gamma_deviance)
|
|
504
546
|
def inner(y, hy):
|
|
505
547
|
return metrics.mean_gamma_deviance(y, hy,
|
|
506
548
|
sample_weight=sample_weight)
|
|
@@ -511,7 +553,7 @@ def mean_gamma_deviance(y_true,
|
|
|
511
553
|
**kwargs)
|
|
512
554
|
|
|
513
555
|
|
|
514
|
-
@
|
|
556
|
+
@metrics_docs(hy_name='y_pred', attr_name='error_func')
|
|
515
557
|
def mean_absolute_percentage_error(y_true,
|
|
516
558
|
*y_pred,
|
|
517
559
|
sample_weight=None,
|
|
@@ -521,6 +563,8 @@ def mean_absolute_percentage_error(y_true,
|
|
|
521
563
|
use_tqdm=True,
|
|
522
564
|
**kwargs):
|
|
523
565
|
"""mean_absolute_percentage_error"""
|
|
566
|
+
|
|
567
|
+
@wraps(metrics.mean_absolute_percentage_error)
|
|
524
568
|
def inner(y, hy):
|
|
525
569
|
return metrics.mean_absolute_percentage_error(y, hy,
|
|
526
570
|
sample_weight=sample_weight,
|
|
@@ -532,7 +576,7 @@ def mean_absolute_percentage_error(y_true,
|
|
|
532
576
|
**kwargs)
|
|
533
577
|
|
|
534
578
|
|
|
535
|
-
@
|
|
579
|
+
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
536
580
|
def d2_absolute_error_score(y_true,
|
|
537
581
|
*y_pred,
|
|
538
582
|
sample_weight=None,
|
|
@@ -542,6 +586,8 @@ def d2_absolute_error_score(y_true,
|
|
|
542
586
|
use_tqdm=True,
|
|
543
587
|
**kwargs):
|
|
544
588
|
"""d2_absolute_error_score"""
|
|
589
|
+
|
|
590
|
+
@wraps(metrics.d2_absolute_error_score)
|
|
545
591
|
def inner(y, hy):
|
|
546
592
|
return metrics.d2_absolute_error_score(y, hy,
|
|
547
593
|
sample_weight=sample_weight,
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
+
import numpy as np
|
|
14
15
|
from sklearn.svm import LinearSVC
|
|
15
16
|
from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
|
|
16
17
|
from sklearn.naive_bayes import GaussianNB
|
|
@@ -19,6 +20,24 @@ from sklearn.model_selection import train_test_split
|
|
|
19
20
|
from sklearn import metrics
|
|
20
21
|
|
|
21
22
|
|
|
23
|
+
def test_difference_f1_score():
|
|
24
|
+
"""Test f1_score"""
|
|
25
|
+
from CompStats.metrics import f1_score
|
|
26
|
+
|
|
27
|
+
X, y = load_iris(return_X_y=True)
|
|
28
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
29
|
+
X_train, X_val, y_train, y_val = _
|
|
30
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
31
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
32
|
+
perf = f1_score(y_val, nb.predict(X_val),
|
|
33
|
+
forest=ens.predict(X_val),
|
|
34
|
+
num_samples=50, average=None)
|
|
35
|
+
diff = perf.difference()
|
|
36
|
+
p_values = diff.p_value(right=False)
|
|
37
|
+
dd = list(p_values.values())[0]
|
|
38
|
+
assert isinstance(dd, np.ndarray)
|
|
39
|
+
print(diff)
|
|
40
|
+
|
|
22
41
|
def test_f1_score():
|
|
23
42
|
"""Test f1_score"""
|
|
24
43
|
from CompStats.metrics import f1_score
|
|
@@ -33,6 +52,11 @@ def test_f1_score():
|
|
|
33
52
|
assert 'forest' in perf.statistic
|
|
34
53
|
_ = metrics.f1_score(y_val, hy, average='macro')
|
|
35
54
|
assert _ == perf.statistic['forest']
|
|
55
|
+
perf = f1_score(y_val, hy, average=None)
|
|
56
|
+
assert str(perf) is not None
|
|
57
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
58
|
+
perf(nb.predict(X_val))
|
|
59
|
+
assert str(perf) is not None
|
|
36
60
|
|
|
37
61
|
|
|
38
62
|
def test_accuracy_score():
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# Copyright 2024 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
from functools import wraps
|
|
15
|
+
try:
|
|
16
|
+
USE_TQDM = True
|
|
17
|
+
from tqdm import tqdm
|
|
18
|
+
except ImportError:
|
|
19
|
+
USE_TQDM = False
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def progress_bar(arg, use_tqdm: bool=True, **kwargs):
|
|
23
|
+
"""Progress bar using tqdm"""
|
|
24
|
+
if not USE_TQDM or not use_tqdm:
|
|
25
|
+
return arg
|
|
26
|
+
return tqdm(arg, **kwargs)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def metrics_docs(hy_name='y_pred', attr_name='score_func'):
|
|
30
|
+
"""Decorator to set docs"""
|
|
31
|
+
|
|
32
|
+
def perf_docs(func):
|
|
33
|
+
"""Decorator to Perf to write :py:class:`~sklearn.metrics` documentation"""
|
|
34
|
+
|
|
35
|
+
func.__doc__ = f""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.{func.__name__}` as :py:attr:`{attr_name}.` The parameters not described can be found in :py:func:`~sklearn.metrics.{func.__name__}`.
|
|
36
|
+
|
|
37
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
38
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
39
|
+
:param {hy_name}: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
|
|
40
|
+
:type {hy_name}: numpy.ndarray
|
|
41
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
42
|
+
:type kwargs: numpy.ndarray
|
|
43
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
44
|
+
:type num_samples: int
|
|
45
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
46
|
+
:type n_jobs: int
|
|
47
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
48
|
+
:type use_tqdm: bool
|
|
49
|
+
|
|
50
|
+
""" + func.__doc__
|
|
51
|
+
|
|
52
|
+
@wraps(func)
|
|
53
|
+
def inner(*args, **kwargs):
|
|
54
|
+
return func(*args, **kwargs)
|
|
55
|
+
|
|
56
|
+
return inner
|
|
57
|
+
return perf_docs
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
# Copyright 2024 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
-
|
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
-
# you may not use this file except in compliance with the License.
|
|
5
|
-
# You may obtain a copy of the License at
|
|
6
|
-
|
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
-
|
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
-
# See the License for the specific language governing permissions and
|
|
13
|
-
# limitations under the License.
|
|
14
|
-
try:
|
|
15
|
-
USE_TQDM = True
|
|
16
|
-
from tqdm import tqdm
|
|
17
|
-
except ImportError:
|
|
18
|
-
USE_TQDM = False
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def progress_bar(arg, use_tqdm: bool=True, **kwargs):
|
|
22
|
-
"""Progress bar using tqdm"""
|
|
23
|
-
if not USE_TQDM or not use_tqdm:
|
|
24
|
-
return arg
|
|
25
|
-
return tqdm(arg, **kwargs)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
from functools import wraps
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
def perf_docs(func):
|
|
32
|
-
"""Decorator to Perf with any write :py:class:`~sklearn.metrics` documentation
|
|
33
|
-
"""
|
|
34
|
-
|
|
35
|
-
func.__doc__ = f""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.{func.__name__}` as :py:attr:`score_func.` The parameters not described can be found in :py:func:`~sklearn.metrics.{func.__name__}`.
|
|
36
|
-
|
|
37
|
-
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
38
|
-
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
39
|
-
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
|
|
40
|
-
:type y_pred: numpy.ndarray
|
|
41
|
-
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
42
|
-
:type kwargs: numpy.ndarray
|
|
43
|
-
:param num_samples: Number of bootstrap samples, default=500.
|
|
44
|
-
:type num_samples: int
|
|
45
|
-
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
46
|
-
:type n_jobs: int
|
|
47
|
-
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
48
|
-
:type use_tqdm: bool
|
|
49
|
-
""" + func.__doc__
|
|
50
|
-
|
|
51
|
-
@wraps(func)
|
|
52
|
-
def inner(*args, **kwargs):
|
|
53
|
-
return func(*args, **kwargs)
|
|
54
|
-
|
|
55
|
-
return inner
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|