CompStats 0.1.3__tar.gz → 0.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/__init__.py +1 -1
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/interface.py +33 -26
- compstats-0.1.4/CompStats/metrics.py +553 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/performance.py +29 -20
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_interface.py +2 -2
- compstats-0.1.4/CompStats/tests/test_metrics.py +453 -0
- compstats-0.1.4/CompStats/utils.py +55 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/PKG-INFO +67 -2
- {compstats-0.1.3 → compstats-0.1.4}/PKG-INFO +67 -2
- compstats-0.1.4/README.rst +93 -0
- compstats-0.1.3/CompStats/metrics.py +0 -197
- compstats-0.1.3/CompStats/tests/test_metrics.py +0 -56
- compstats-0.1.3/CompStats/utils.py +0 -25
- compstats-0.1.3/README.rst +0 -28
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/measurements.py +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/LICENSE +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/MANIFEST.in +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/pyproject.toml +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/setup.cfg +0 -0
- {compstats-0.1.3 → compstats-0.1.4}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.4'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -31,8 +31,8 @@ class Perf(object):
|
|
|
31
31
|
:type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
32
32
|
:param error_func: Function to measure the performance where the best algorithm has the lowest value.
|
|
33
33
|
:type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
34
|
-
:param
|
|
35
|
-
:type
|
|
34
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
35
|
+
:type y_pred: numpy.ndarray
|
|
36
36
|
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
37
37
|
:type kwargs: numpy.ndarray
|
|
38
38
|
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
@@ -59,19 +59,21 @@ class Perf(object):
|
|
|
59
59
|
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
60
60
|
>>> perf
|
|
61
61
|
<Perf>
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
Statistic with its standard error (se)
|
|
63
|
+
statistic (se)
|
|
64
|
+
0.9792 (0.0221) <= alg-1
|
|
65
|
+
0.9744 (0.0246) <= forest
|
|
65
66
|
|
|
66
67
|
If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
|
|
67
68
|
|
|
68
69
|
>>> lr = LogisticRegression().fit(X_train, y_train)
|
|
69
70
|
>>> perf(lr.predict(X_val), name='Log. Reg.')
|
|
70
71
|
<Perf>
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
Statistic with its standard error (se)
|
|
73
|
+
statistic (se)
|
|
74
|
+
1.0000 (0.0000) <= Log. Reg.
|
|
75
|
+
0.9792 (0.0221) <= alg-1
|
|
76
|
+
0.9744 (0.0246) <= forest
|
|
75
77
|
|
|
76
78
|
The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
|
|
77
79
|
|
|
@@ -79,13 +81,14 @@ class Perf(object):
|
|
|
79
81
|
>>> perf_error.error_func = lambda y, hy: (y != hy).mean()
|
|
80
82
|
>>> perf_error
|
|
81
83
|
<Perf>
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
84
|
+
Statistic with its standard error (se)
|
|
85
|
+
statistic (se)
|
|
86
|
+
0.0000 (0.0000) <= Log. Reg.
|
|
87
|
+
0.0222 (0.0237) <= alg-1
|
|
88
|
+
0.0222 (0.0215) <= forest
|
|
86
89
|
|
|
87
90
|
"""
|
|
88
|
-
def __init__(self, y_true, *
|
|
91
|
+
def __init__(self, y_true, *y_pred,
|
|
89
92
|
score_func=balanced_accuracy_score,
|
|
90
93
|
error_func=None,
|
|
91
94
|
num_samples: int=500,
|
|
@@ -96,7 +99,7 @@ class Perf(object):
|
|
|
96
99
|
self.score_func = score_func
|
|
97
100
|
self.error_func = error_func
|
|
98
101
|
algs = {}
|
|
99
|
-
for k, v in enumerate(
|
|
102
|
+
for k, v in enumerate(y_pred):
|
|
100
103
|
algs[f'alg-{k+1}'] = np.asanyarray(v)
|
|
101
104
|
algs.update(**kwargs)
|
|
102
105
|
self.predictions = algs
|
|
@@ -137,7 +140,7 @@ class Perf(object):
|
|
|
137
140
|
ins.predictions = dict(self.predictions)
|
|
138
141
|
ins._statistic_samples._samples = self.statistic_samples._samples
|
|
139
142
|
return ins
|
|
140
|
-
|
|
143
|
+
|
|
141
144
|
def __repr__(self):
|
|
142
145
|
"""Prediction statistics with standard error in parenthesis"""
|
|
143
146
|
return f"<{self.__class__.__name__}>\n{self}"
|
|
@@ -145,10 +148,11 @@ class Perf(object):
|
|
|
145
148
|
def __str__(self):
|
|
146
149
|
"""Prediction statistics with standard error in parenthesis"""
|
|
147
150
|
|
|
148
|
-
se = self.se
|
|
149
|
-
output = ["
|
|
150
|
-
|
|
151
|
-
|
|
151
|
+
se = self.se
|
|
152
|
+
output = ["Statistic with its standard error (se)"]
|
|
153
|
+
output.append("statistic (se)")
|
|
154
|
+
for key, value in self.statistic.items():
|
|
155
|
+
output.append(f'{value:0.4f} ({se[key]:0.4f}) <= {key}')
|
|
152
156
|
return "\n".join(output)
|
|
153
157
|
|
|
154
158
|
def __call__(self, y_pred, name=None):
|
|
@@ -196,7 +200,7 @@ class Perf(object):
|
|
|
196
200
|
continue
|
|
197
201
|
diff[k] = sign * (base - v)
|
|
198
202
|
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
199
|
-
statistic=self.statistic
|
|
203
|
+
statistic=self.statistic,
|
|
200
204
|
best=self.best[0])
|
|
201
205
|
diff_ins.statistic_samples.calls = diff
|
|
202
206
|
diff_ins.statistic_samples.info['best'] = self.best[0]
|
|
@@ -209,12 +213,13 @@ class Perf(object):
|
|
|
209
213
|
try:
|
|
210
214
|
return self._best
|
|
211
215
|
except AttributeError:
|
|
212
|
-
statistic = [(k, v) for k, v in self.statistic
|
|
216
|
+
statistic = [(k, v) for k, v in self.statistic.items()]
|
|
213
217
|
statistic = sorted(statistic, key=lambda x: x[1],
|
|
214
218
|
reverse=self.statistic_samples.BiB)
|
|
215
219
|
self._best = statistic[0]
|
|
216
220
|
return self._best
|
|
217
221
|
|
|
222
|
+
@property
|
|
218
223
|
def statistic(self):
|
|
219
224
|
"""Statistic
|
|
220
225
|
|
|
@@ -230,7 +235,7 @@ class Perf(object):
|
|
|
230
235
|
>>> hy = m.predict(X_val)
|
|
231
236
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
232
237
|
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
233
|
-
>>> perf.statistic
|
|
238
|
+
>>> perf.statistic
|
|
234
239
|
{'alg-1': 1.0, 'forest': 0.9500891265597148}
|
|
235
240
|
"""
|
|
236
241
|
|
|
@@ -239,6 +244,7 @@ class Perf(object):
|
|
|
239
244
|
key=lambda x: x[1], reverse=self.statistic_samples.BiB)
|
|
240
245
|
return dict(data)
|
|
241
246
|
|
|
247
|
+
@property
|
|
242
248
|
def se(self):
|
|
243
249
|
"""Standard Error
|
|
244
250
|
|
|
@@ -254,9 +260,10 @@ class Perf(object):
|
|
|
254
260
|
>>> hy = m.predict(X_val)
|
|
255
261
|
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
256
262
|
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
257
|
-
>>>
|
|
263
|
+
>>> perf.se
|
|
258
264
|
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
259
265
|
"""
|
|
266
|
+
|
|
260
267
|
return SE(self.statistic_samples)
|
|
261
268
|
|
|
262
269
|
def plot(self, **kwargs):
|
|
@@ -405,7 +412,7 @@ class Difference:
|
|
|
405
412
|
>>> diff
|
|
406
413
|
<Difference>
|
|
407
414
|
difference p-values w.r.t alg-1
|
|
408
|
-
|
|
415
|
+
0.0780 <= forest
|
|
409
416
|
"""
|
|
410
417
|
|
|
411
418
|
statistic_samples:StatisticSamples=None
|
|
@@ -420,7 +427,7 @@ class Difference:
|
|
|
420
427
|
"""p-value"""
|
|
421
428
|
output = [f"difference p-values w.r.t {self.best}"]
|
|
422
429
|
for k, v in self.p_value().items():
|
|
423
|
-
output.append(f'{
|
|
430
|
+
output.append(f'{v:0.4f} <= {k}')
|
|
424
431
|
return "\n".join(output)
|
|
425
432
|
|
|
426
433
|
def p_value(self):
|