CompStats 0.1.2__tar.gz → 0.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/__init__.py +2 -1
- compstats-0.1.3/CompStats/interface.py +474 -0
- compstats-0.1.3/CompStats/metrics.py +197 -0
- compstats-0.1.3/CompStats/tests/test_interface.py +153 -0
- compstats-0.1.3/CompStats/tests/test_metrics.py +56 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/tests/test_performance.py +2 -2
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/utils.py +4 -4
- {compstats-0.1.2 → compstats-0.1.3}/CompStats.egg-info/PKG-INFO +2 -2
- {compstats-0.1.2 → compstats-0.1.3}/CompStats.egg-info/SOURCES.txt +4 -0
- {compstats-0.1.2 → compstats-0.1.3}/PKG-INFO +2 -2
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/measurements.py +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/performance.py +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/LICENSE +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/MANIFEST.in +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/README.rst +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/pyproject.toml +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/setup.cfg +0 -0
- {compstats-0.1.2 → compstats-0.1.3}/setup.py +0 -0
|
@@ -11,9 +11,10 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.3'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
18
18
|
from CompStats.performance import performance_multiple_metrics, difference_multiple, plot_performance_multiple, plot_difference_multiple
|
|
19
19
|
from CompStats.performance import all_differences_multiple, plot_performance2, plot_difference2, plot_scatter_matrix
|
|
20
|
+
from CompStats.interface import Perf
|
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
# Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from sklearn.metrics import balanced_accuracy_score
|
|
16
|
+
from sklearn.base import clone
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import numpy as np
|
|
19
|
+
from CompStats.bootstrap import StatisticSamples
|
|
20
|
+
from CompStats.utils import progress_bar
|
|
21
|
+
from CompStats.measurements import SE
|
|
22
|
+
from CompStats.performance import plot_performance, plot_difference
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Perf(object):
|
|
26
|
+
"""Perf is an entry point to CompStats
|
|
27
|
+
|
|
28
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
29
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
30
|
+
:param score_func: Function to measure the performance, it is assumed that the best algorithm has the highest value.
|
|
31
|
+
:type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
32
|
+
:param error_func: Function to measure the performance where the best algorithm has the lowest value.
|
|
33
|
+
:type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
|
|
34
|
+
:param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
35
|
+
:type args: numpy.ndarray
|
|
36
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
37
|
+
:type kwargs: numpy.ndarray
|
|
38
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
39
|
+
:type n_jobs: int
|
|
40
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
41
|
+
:type num_samples: int
|
|
42
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True.
|
|
43
|
+
:type use_tqdm: bool
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
>>> from sklearn.svm import LinearSVC
|
|
47
|
+
>>> from sklearn.linear_model import LogisticRegression
|
|
48
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
49
|
+
>>> from sklearn.datasets import load_iris
|
|
50
|
+
>>> from sklearn.model_selection import train_test_split
|
|
51
|
+
>>> from sklearn.base import clone
|
|
52
|
+
>>> from CompStats.interface import Perf
|
|
53
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
54
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
55
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
56
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
57
|
+
>>> hy = m.predict(X_val)
|
|
58
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
59
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
60
|
+
>>> perf
|
|
61
|
+
<Perf>
|
|
62
|
+
Prediction statistics with standard error
|
|
63
|
+
alg-1 = 1.000 (0.000)
|
|
64
|
+
forest = 0.946 (0.038)
|
|
65
|
+
|
|
66
|
+
If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
|
|
67
|
+
|
|
68
|
+
>>> lr = LogisticRegression().fit(X_train, y_train)
|
|
69
|
+
>>> perf(lr.predict(X_val), name='Log. Reg.')
|
|
70
|
+
<Perf>
|
|
71
|
+
Prediction statistics with standard error
|
|
72
|
+
alg-1 = 1.000 (0.000)
|
|
73
|
+
forest = 0.946 (0.038)
|
|
74
|
+
Log. Reg. = 0.946 (0.038)
|
|
75
|
+
|
|
76
|
+
The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
|
|
77
|
+
|
|
78
|
+
>>> perf_error = clone(perf)
|
|
79
|
+
>>> perf_error.error_func = lambda y, hy: (y != hy).mean()
|
|
80
|
+
>>> perf_error
|
|
81
|
+
<Perf>
|
|
82
|
+
Prediction statistics with standard error
|
|
83
|
+
alg-1 = 0.000 (0.000)
|
|
84
|
+
forest = 0.044 (0.030)
|
|
85
|
+
Log. Reg. = 0.044 (0.030)
|
|
86
|
+
|
|
87
|
+
"""
|
|
88
|
+
def __init__(self, y_true, *args,
|
|
89
|
+
score_func=balanced_accuracy_score,
|
|
90
|
+
error_func=None,
|
|
91
|
+
num_samples: int=500,
|
|
92
|
+
n_jobs: int=-1,
|
|
93
|
+
use_tqdm=True,
|
|
94
|
+
**kwargs):
|
|
95
|
+
assert (score_func is None) ^ (error_func is None)
|
|
96
|
+
self.score_func = score_func
|
|
97
|
+
self.error_func = error_func
|
|
98
|
+
algs = {}
|
|
99
|
+
for k, v in enumerate(args):
|
|
100
|
+
algs[f'alg-{k+1}'] = np.asanyarray(v)
|
|
101
|
+
algs.update(**kwargs)
|
|
102
|
+
self.predictions = algs
|
|
103
|
+
self.y_true = y_true
|
|
104
|
+
self.num_samples = num_samples
|
|
105
|
+
self.n_jobs = n_jobs
|
|
106
|
+
self.use_tqdm = use_tqdm
|
|
107
|
+
self._init()
|
|
108
|
+
|
|
109
|
+
def _init(self):
|
|
110
|
+
"""Compute the bootstrap statistic"""
|
|
111
|
+
|
|
112
|
+
bib = True if self.score_func is not None else False
|
|
113
|
+
if hasattr(self, '_statistic_samples'):
|
|
114
|
+
_ = self.statistic_samples
|
|
115
|
+
_.BiB = bib
|
|
116
|
+
else:
|
|
117
|
+
_ = StatisticSamples(statistic=self.statistic_func,
|
|
118
|
+
n_jobs=self.n_jobs,
|
|
119
|
+
num_samples=self.num_samples,
|
|
120
|
+
BiB=bib)
|
|
121
|
+
_.samples(N=self.y_true.shape[0])
|
|
122
|
+
self.statistic_samples = _
|
|
123
|
+
|
|
124
|
+
def get_params(self):
|
|
125
|
+
"""Parameters"""
|
|
126
|
+
|
|
127
|
+
return dict(y_true=self.y_true,
|
|
128
|
+
score_func=self.score_func,
|
|
129
|
+
error_func=self.error_func,
|
|
130
|
+
num_samples=self.num_samples,
|
|
131
|
+
n_jobs=self.n_jobs)
|
|
132
|
+
|
|
133
|
+
def __sklearn_clone__(self):
|
|
134
|
+
klass = self.__class__
|
|
135
|
+
params = self.get_params()
|
|
136
|
+
ins = klass(**params)
|
|
137
|
+
ins.predictions = dict(self.predictions)
|
|
138
|
+
ins._statistic_samples._samples = self.statistic_samples._samples
|
|
139
|
+
return ins
|
|
140
|
+
|
|
141
|
+
def __repr__(self):
|
|
142
|
+
"""Prediction statistics with standard error in parenthesis"""
|
|
143
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
144
|
+
|
|
145
|
+
def __str__(self):
|
|
146
|
+
"""Prediction statistics with standard error in parenthesis"""
|
|
147
|
+
|
|
148
|
+
se = self.se()
|
|
149
|
+
output = ["Prediction statistics with standard error"]
|
|
150
|
+
for key, value in self.statistic().items():
|
|
151
|
+
output.append(f'{key} = {value:0.3f} ({se[key]:0.3f})')
|
|
152
|
+
return "\n".join(output)
|
|
153
|
+
|
|
154
|
+
def __call__(self, y_pred, name=None):
|
|
155
|
+
"""Add predictions"""
|
|
156
|
+
if name is None:
|
|
157
|
+
k = len(self.predictions) + 1
|
|
158
|
+
if k == 0:
|
|
159
|
+
k = 1
|
|
160
|
+
name = f'alg-{k}'
|
|
161
|
+
self.predictions[name] = np.asanyarray(y_pred)
|
|
162
|
+
samples = self._statistic_samples
|
|
163
|
+
calls = samples.calls
|
|
164
|
+
if name in calls:
|
|
165
|
+
del calls[name]
|
|
166
|
+
return self
|
|
167
|
+
|
|
168
|
+
def difference(self, wrt_to: str=None):
|
|
169
|
+
"""Compute the difference w.r.t any algorithm by default is the best
|
|
170
|
+
|
|
171
|
+
>>> from sklearn.svm import LinearSVC
|
|
172
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
173
|
+
>>> from sklearn.datasets import load_iris
|
|
174
|
+
>>> from sklearn.model_selection import train_test_split
|
|
175
|
+
>>> from sklearn.base import clone
|
|
176
|
+
>>> from CompStats.interface import Perf
|
|
177
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
178
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
179
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
180
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
181
|
+
>>> hy = m.predict(X_val)
|
|
182
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
183
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
184
|
+
>>> perf.difference()
|
|
185
|
+
<Difference>
|
|
186
|
+
difference p-values w.r.t alg-1
|
|
187
|
+
forest 0.06
|
|
188
|
+
"""
|
|
189
|
+
if wrt_to is None:
|
|
190
|
+
wrt_to = self.best[0]
|
|
191
|
+
base = self.statistic_samples.calls[wrt_to]
|
|
192
|
+
sign = 1 if self.statistic_samples.BiB else -1
|
|
193
|
+
diff = dict()
|
|
194
|
+
for k, v in self.statistic_samples.calls.items():
|
|
195
|
+
if k == wrt_to:
|
|
196
|
+
continue
|
|
197
|
+
diff[k] = sign * (base - v)
|
|
198
|
+
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
199
|
+
statistic=self.statistic(),
|
|
200
|
+
best=self.best[0])
|
|
201
|
+
diff_ins.statistic_samples.calls = diff
|
|
202
|
+
diff_ins.statistic_samples.info['best'] = self.best[0]
|
|
203
|
+
return diff_ins
|
|
204
|
+
|
|
205
|
+
@property
|
|
206
|
+
def best(self):
|
|
207
|
+
"""Best system"""
|
|
208
|
+
|
|
209
|
+
try:
|
|
210
|
+
return self._best
|
|
211
|
+
except AttributeError:
|
|
212
|
+
statistic = [(k, v) for k, v in self.statistic().items()]
|
|
213
|
+
statistic = sorted(statistic, key=lambda x: x[1],
|
|
214
|
+
reverse=self.statistic_samples.BiB)
|
|
215
|
+
self._best = statistic[0]
|
|
216
|
+
return self._best
|
|
217
|
+
|
|
218
|
+
def statistic(self):
|
|
219
|
+
"""Statistic
|
|
220
|
+
|
|
221
|
+
>>> from sklearn.svm import LinearSVC
|
|
222
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
223
|
+
>>> from sklearn.datasets import load_iris
|
|
224
|
+
>>> from sklearn.model_selection import train_test_split
|
|
225
|
+
>>> from CompStats.interface import Perf
|
|
226
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
227
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
228
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
229
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
230
|
+
>>> hy = m.predict(X_val)
|
|
231
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
232
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
233
|
+
>>> perf.statistic()
|
|
234
|
+
{'alg-1': 1.0, 'forest': 0.9500891265597148}
|
|
235
|
+
"""
|
|
236
|
+
|
|
237
|
+
data = sorted([(k, self.statistic_func(self.y_true, v))
|
|
238
|
+
for k, v in self.predictions.items()],
|
|
239
|
+
key=lambda x: x[1], reverse=self.statistic_samples.BiB)
|
|
240
|
+
return dict(data)
|
|
241
|
+
|
|
242
|
+
def se(self):
|
|
243
|
+
"""Standard Error
|
|
244
|
+
|
|
245
|
+
>>> from sklearn.svm import LinearSVC
|
|
246
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
247
|
+
>>> from sklearn.datasets import load_iris
|
|
248
|
+
>>> from sklearn.model_selection import train_test_split
|
|
249
|
+
>>> from CompStats.interface import Perf
|
|
250
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
251
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
252
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
253
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
254
|
+
>>> hy = m.predict(X_val)
|
|
255
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
256
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
257
|
+
>>> print(perf.se())
|
|
258
|
+
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
259
|
+
"""
|
|
260
|
+
return SE(self.statistic_samples)
|
|
261
|
+
|
|
262
|
+
def plot(self, **kwargs):
|
|
263
|
+
"""plot with seaborn
|
|
264
|
+
|
|
265
|
+
>>> from sklearn.svm import LinearSVC
|
|
266
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
267
|
+
>>> from sklearn.datasets import load_iris
|
|
268
|
+
>>> from sklearn.model_selection import train_test_split
|
|
269
|
+
>>> from CompStats.interface import Perf
|
|
270
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
271
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
272
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
273
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
274
|
+
>>> hy = m.predict(X_val)
|
|
275
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
276
|
+
>>> perf = Perf(y_val, hy, score_func=None,
|
|
277
|
+
error_func=lambda y, hy: (y != hy).mean(),
|
|
278
|
+
forest=ens.predict(X_val))
|
|
279
|
+
>>> perf.plot()
|
|
280
|
+
"""
|
|
281
|
+
if self.score_func is not None:
|
|
282
|
+
value_name = 'Score'
|
|
283
|
+
else:
|
|
284
|
+
value_name = 'Error'
|
|
285
|
+
_ = dict(value_name=value_name)
|
|
286
|
+
_.update(kwargs)
|
|
287
|
+
return plot_performance(self.statistic_samples, **_)
|
|
288
|
+
|
|
289
|
+
@property
|
|
290
|
+
def n_jobs(self):
|
|
291
|
+
"""Number of jobs to compute the statistics"""
|
|
292
|
+
return self._n_jobs
|
|
293
|
+
|
|
294
|
+
@n_jobs.setter
|
|
295
|
+
def n_jobs(self, value):
|
|
296
|
+
self._n_jobs = value
|
|
297
|
+
|
|
298
|
+
@property
|
|
299
|
+
def statistic_func(self):
|
|
300
|
+
"""Statistic function"""
|
|
301
|
+
if self.score_func is not None:
|
|
302
|
+
return self.score_func
|
|
303
|
+
return self.error_func
|
|
304
|
+
|
|
305
|
+
@property
|
|
306
|
+
def statistic_samples(self):
|
|
307
|
+
"""Statistic Samples"""
|
|
308
|
+
|
|
309
|
+
samples = self._statistic_samples
|
|
310
|
+
algs = set(samples.calls.keys())
|
|
311
|
+
algs = set(self.predictions.keys()) - algs
|
|
312
|
+
if len(algs):
|
|
313
|
+
for key in progress_bar(algs, use_tqdm=self.use_tqdm):
|
|
314
|
+
samples(self.y_true, self.predictions[key], name=key)
|
|
315
|
+
return self._statistic_samples
|
|
316
|
+
|
|
317
|
+
@statistic_samples.setter
|
|
318
|
+
def statistic_samples(self, value):
|
|
319
|
+
self._statistic_samples = value
|
|
320
|
+
|
|
321
|
+
@property
|
|
322
|
+
def num_samples(self):
|
|
323
|
+
"""Number of bootstrap samples"""
|
|
324
|
+
return self._num_samples
|
|
325
|
+
|
|
326
|
+
@num_samples.setter
|
|
327
|
+
def num_samples(self, value):
|
|
328
|
+
self._num_samples = value
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
def predictions(self):
|
|
332
|
+
"""Predictions"""
|
|
333
|
+
return self._predictions
|
|
334
|
+
|
|
335
|
+
@predictions.setter
|
|
336
|
+
def predictions(self, value):
|
|
337
|
+
self._predictions = value
|
|
338
|
+
|
|
339
|
+
@property
|
|
340
|
+
def y_true(self):
|
|
341
|
+
"""True output, gold standard o :math:`y`"""
|
|
342
|
+
|
|
343
|
+
return self._y_true
|
|
344
|
+
|
|
345
|
+
@y_true.setter
|
|
346
|
+
def y_true(self, value):
|
|
347
|
+
if isinstance(value, pd.DataFrame):
|
|
348
|
+
self._y_true = value['y'].to_numpy()
|
|
349
|
+
algs = {}
|
|
350
|
+
for c in value.columns:
|
|
351
|
+
if c == 'y':
|
|
352
|
+
continue
|
|
353
|
+
algs[c] = value[c].to_numpy()
|
|
354
|
+
self.predictions.update(algs)
|
|
355
|
+
return
|
|
356
|
+
self._y_true = value
|
|
357
|
+
|
|
358
|
+
@property
|
|
359
|
+
def score_func(self):
|
|
360
|
+
"""Score function"""
|
|
361
|
+
return self._score_func
|
|
362
|
+
|
|
363
|
+
@score_func.setter
|
|
364
|
+
def score_func(self, value):
|
|
365
|
+
self._score_func = value
|
|
366
|
+
if value is not None:
|
|
367
|
+
self.error_func = None
|
|
368
|
+
if hasattr(self, '_statistic_samples'):
|
|
369
|
+
self._statistic_samples.statistic = value
|
|
370
|
+
self._statistic_samples.BiB = True
|
|
371
|
+
|
|
372
|
+
@property
|
|
373
|
+
def error_func(self):
|
|
374
|
+
"""Error function"""
|
|
375
|
+
return self._error_func
|
|
376
|
+
|
|
377
|
+
@error_func.setter
|
|
378
|
+
def error_func(self, value):
|
|
379
|
+
self._error_func = value
|
|
380
|
+
if value is not None:
|
|
381
|
+
self.score_func = None
|
|
382
|
+
if hasattr(self, '_statistic_samples'):
|
|
383
|
+
self._statistic_samples.statistic = value
|
|
384
|
+
self._statistic_samples.BiB = False
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
@dataclass
|
|
388
|
+
class Difference:
|
|
389
|
+
"""Difference
|
|
390
|
+
|
|
391
|
+
>>> from sklearn.svm import LinearSVC
|
|
392
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
393
|
+
>>> from sklearn.datasets import load_iris
|
|
394
|
+
>>> from sklearn.model_selection import train_test_split
|
|
395
|
+
>>> from sklearn.base import clone
|
|
396
|
+
>>> from CompStats.interface import Perf
|
|
397
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
398
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
399
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
400
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
401
|
+
>>> hy = m.predict(X_val)
|
|
402
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
403
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
404
|
+
>>> diff = perf.difference()
|
|
405
|
+
>>> diff
|
|
406
|
+
<Difference>
|
|
407
|
+
difference p-values w.r.t alg-1
|
|
408
|
+
forest 0.3
|
|
409
|
+
"""
|
|
410
|
+
|
|
411
|
+
statistic_samples:StatisticSamples=None
|
|
412
|
+
best:str=None
|
|
413
|
+
statistic:dict=None
|
|
414
|
+
|
|
415
|
+
def __repr__(self):
|
|
416
|
+
"""p-value"""
|
|
417
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
418
|
+
|
|
419
|
+
def __str__(self):
|
|
420
|
+
"""p-value"""
|
|
421
|
+
output = [f"difference p-values w.r.t {self.best}"]
|
|
422
|
+
for k, v in self.p_value().items():
|
|
423
|
+
output.append(f'{k} {v}')
|
|
424
|
+
return "\n".join(output)
|
|
425
|
+
|
|
426
|
+
def p_value(self):
|
|
427
|
+
"""Compute p_value of the differences
|
|
428
|
+
|
|
429
|
+
>>> from sklearn.svm import LinearSVC
|
|
430
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
431
|
+
>>> from sklearn.datasets import load_iris
|
|
432
|
+
>>> from sklearn.model_selection import train_test_split
|
|
433
|
+
>>> from sklearn.base import clone
|
|
434
|
+
>>> from CompStats.interface import Perf
|
|
435
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
436
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
437
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
438
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
439
|
+
>>> hy = m.predict(X_val)
|
|
440
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
441
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
442
|
+
>>> diff = perf.difference()
|
|
443
|
+
>>> diff.p_value()
|
|
444
|
+
{'forest': np.float64(0.3)}
|
|
445
|
+
"""
|
|
446
|
+
values = []
|
|
447
|
+
sign = 1 if self.statistic_samples.BiB else -1
|
|
448
|
+
for k, v in self.statistic_samples.calls.items():
|
|
449
|
+
delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
|
|
450
|
+
values.append((k, (v > delta).mean()))
|
|
451
|
+
values.sort(key=lambda x: x[1])
|
|
452
|
+
return dict(values)
|
|
453
|
+
|
|
454
|
+
def plot(self, **kwargs):
|
|
455
|
+
"""Plot
|
|
456
|
+
|
|
457
|
+
>>> from sklearn.svm import LinearSVC
|
|
458
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
459
|
+
>>> from sklearn.datasets import load_iris
|
|
460
|
+
>>> from sklearn.model_selection import train_test_split
|
|
461
|
+
>>> from sklearn.base import clone
|
|
462
|
+
>>> from CompStats.interface import Perf
|
|
463
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
464
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
465
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
466
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
467
|
+
>>> hy = m.predict(X_val)
|
|
468
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
469
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
470
|
+
>>> diff = perf.difference()
|
|
471
|
+
>>> diff.plot()
|
|
472
|
+
"""
|
|
473
|
+
|
|
474
|
+
return plot_difference(self.statistic_samples, **kwargs)
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
from CompStats.interface import Perf
|
|
15
|
+
from sklearn import metrics
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def accuracy_score(y_true, *args,
|
|
19
|
+
normalize=True, sample_weight=None,
|
|
20
|
+
num_samples: int=500,
|
|
21
|
+
n_jobs: int=-1,
|
|
22
|
+
use_tqdm=True,
|
|
23
|
+
**kwargs):
|
|
24
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`sklearn.metrics.accuracy_score` as :py:attr:`score_func.`
|
|
25
|
+
|
|
26
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
27
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
28
|
+
:param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
29
|
+
:type args: numpy.ndarray
|
|
30
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
31
|
+
:type args: numpy.ndarray
|
|
32
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
33
|
+
:type num_samples: int
|
|
34
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
35
|
+
:type n_jobs: int
|
|
36
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
37
|
+
:type use_tqdm: bool
|
|
38
|
+
:param normalize: see sklearn.metrics.f1_score
|
|
39
|
+
:param sample_weight: see sklearn.metrics.f1_score
|
|
40
|
+
|
|
41
|
+
>>> from sklearn.svm import LinearSVC
|
|
42
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
43
|
+
>>> from sklearn.datasets import load_iris
|
|
44
|
+
>>> from sklearn.model_selection import train_test_split
|
|
45
|
+
>>> from sklearn.base import clone
|
|
46
|
+
>>> from CompStats.metrics import accuracy_score
|
|
47
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
48
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
49
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
50
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
51
|
+
>>> hy = m.predict(X_val)
|
|
52
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
53
|
+
>>> score = accuracy_score(y_val, hy,
|
|
54
|
+
forest=ens.predict(X_val))
|
|
55
|
+
>>> score
|
|
56
|
+
<Perf>
|
|
57
|
+
Prediction statistics with standard error
|
|
58
|
+
forest = 0.978 (0.023)
|
|
59
|
+
alg-1 = 0.956 (0.030)
|
|
60
|
+
>>> diff = score.difference()
|
|
61
|
+
>>> diff
|
|
62
|
+
<Difference>
|
|
63
|
+
difference p-values w.r.t forest
|
|
64
|
+
alg-1 0.252
|
|
65
|
+
"""
|
|
66
|
+
|
|
67
|
+
def inner(y, hy):
|
|
68
|
+
return metrics.accuracy_score(y, hy,
|
|
69
|
+
normalize=normalize,
|
|
70
|
+
sample_weight=sample_weight)
|
|
71
|
+
return Perf(y_true, *args, score_func=inner,
|
|
72
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
73
|
+
use_tqdm=use_tqdm,
|
|
74
|
+
**kwargs)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def balanced_accuracy_score(y_true, *args,
|
|
78
|
+
sample_weight=None, adjusted=False,
|
|
79
|
+
num_samples: int=500,
|
|
80
|
+
n_jobs: int=-1,
|
|
81
|
+
use_tqdm=True,
|
|
82
|
+
**kwargs):
|
|
83
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`sklearn.metrics.balanced_accuracy_score` as :py:attr:`score_func.`
|
|
84
|
+
|
|
85
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
86
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
87
|
+
:param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
88
|
+
:type args: numpy.ndarray
|
|
89
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
90
|
+
:type args: numpy.ndarray
|
|
91
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
92
|
+
:type num_samples: int
|
|
93
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
94
|
+
:type n_jobs: int
|
|
95
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
96
|
+
:type use_tqdm: bool
|
|
97
|
+
:param sample_weight: see sklearn.metrics.f1_score
|
|
98
|
+
:param adjusted: see sklearn.metrics.f1_score
|
|
99
|
+
|
|
100
|
+
>>> from sklearn.svm import LinearSVC
|
|
101
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
102
|
+
>>> from sklearn.datasets import load_iris
|
|
103
|
+
>>> from sklearn.model_selection import train_test_split
|
|
104
|
+
>>> from sklearn.base import clone
|
|
105
|
+
>>> from CompStats.metrics import balanced_accuracy_score
|
|
106
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
107
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
108
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
109
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
110
|
+
>>> hy = m.predict(X_val)
|
|
111
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
112
|
+
>>> score = balanced_accuracy_score(y_val, hy,
|
|
113
|
+
forest=ens.predict(X_val))
|
|
114
|
+
>>> score
|
|
115
|
+
<Perf>
|
|
116
|
+
Prediction statistics with standard error
|
|
117
|
+
forest = 0.957 (0.031)
|
|
118
|
+
alg-1 = 0.935 (0.037)
|
|
119
|
+
>>> diff = score.difference()
|
|
120
|
+
>>> diff
|
|
121
|
+
<Difference>
|
|
122
|
+
difference p-values w.r.t forest
|
|
123
|
+
alg-1 0.254
|
|
124
|
+
"""
|
|
125
|
+
|
|
126
|
+
def inner(y, hy):
|
|
127
|
+
return metrics.balanced_accuracy_score(y, hy,
|
|
128
|
+
adjusted=adjusted,
|
|
129
|
+
sample_weight=sample_weight)
|
|
130
|
+
return Perf(y_true, *args, score_func=inner,
|
|
131
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
132
|
+
use_tqdm=use_tqdm,
|
|
133
|
+
**kwargs)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def f1_score(y_true, *args, labels=None, pos_label=1,
|
|
137
|
+
average='binary', sample_weight=None,
|
|
138
|
+
zero_division='warn', num_samples: int=500,
|
|
139
|
+
n_jobs: int=-1, use_tqdm=True,
|
|
140
|
+
**kwargs):
|
|
141
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`sklearn.metrics.f1_score` as :py:attr:`score_func.`
|
|
142
|
+
|
|
143
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
144
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
145
|
+
:param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
|
|
146
|
+
:type args: numpy.ndarray
|
|
147
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
148
|
+
:type args: numpy.ndarray
|
|
149
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
150
|
+
:type num_samples: int
|
|
151
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
152
|
+
:type n_jobs: int
|
|
153
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
154
|
+
:type use_tqdm: bool
|
|
155
|
+
:param labels: see sklearn.metrics.f1_score
|
|
156
|
+
:param pos_label: see sklearn.metrics.f1_score
|
|
157
|
+
:param average: see sklearn.metrics.f1_score
|
|
158
|
+
:param sample_weight: see sklearn.metrics.f1_score
|
|
159
|
+
:param zero_division: see sklearn.metrics.f1_score
|
|
160
|
+
|
|
161
|
+
>>> from sklearn.svm import LinearSVC
|
|
162
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
163
|
+
>>> from sklearn.datasets import load_iris
|
|
164
|
+
>>> from sklearn.model_selection import train_test_split
|
|
165
|
+
>>> from sklearn.base import clone
|
|
166
|
+
>>> from CompStats.metrics import f1_score
|
|
167
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
168
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
169
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
170
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
171
|
+
>>> hy = m.predict(X_val)
|
|
172
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
173
|
+
>>> score = f1_score(y_val, hy,
|
|
174
|
+
forest=ens.predict(X_val),
|
|
175
|
+
average='macro')
|
|
176
|
+
>>> score
|
|
177
|
+
<Perf>
|
|
178
|
+
Prediction statistics with standard error
|
|
179
|
+
forest = 0.954 (0.032)
|
|
180
|
+
alg-1 = 0.931 (0.040)
|
|
181
|
+
>>> diff = score.difference()
|
|
182
|
+
>>> diff
|
|
183
|
+
<Difference>
|
|
184
|
+
difference p-values w.r.t forest
|
|
185
|
+
alg-1 0.176
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
def inner(y, hy):
|
|
189
|
+
return metrics.f1_score(y, hy, labels=labels,
|
|
190
|
+
pos_label=pos_label,
|
|
191
|
+
average=average,
|
|
192
|
+
sample_weight=sample_weight,
|
|
193
|
+
zero_division=zero_division)
|
|
194
|
+
return Perf(y_true, *args, score_func=inner,
|
|
195
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
196
|
+
use_tqdm=use_tqdm,
|
|
197
|
+
**kwargs)
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from sklearn.base import clone
|
|
17
|
+
from sklearn.svm import LinearSVC
|
|
18
|
+
from sklearn.ensemble import RandomForestClassifier
|
|
19
|
+
from sklearn.datasets import load_iris
|
|
20
|
+
from sklearn.model_selection import train_test_split
|
|
21
|
+
import pandas as pd
|
|
22
|
+
from CompStats.tests.test_performance import DATA
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def test_Perf():
|
|
26
|
+
"""Test perf"""
|
|
27
|
+
from CompStats.interface import Perf
|
|
28
|
+
|
|
29
|
+
X, y = load_iris(return_X_y=True)
|
|
30
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
31
|
+
X_train, X_val, y_train, y_val = _
|
|
32
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
33
|
+
hy = m.predict(X_val)
|
|
34
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
35
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
36
|
+
assert 'alg-1' in perf.predictions
|
|
37
|
+
assert 'forest' in perf.predictions
|
|
38
|
+
assert str(perf) is not None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_Perf_statistic():
|
|
42
|
+
"""Test statistic"""
|
|
43
|
+
from CompStats.interface import Perf
|
|
44
|
+
|
|
45
|
+
X, y = load_iris(return_X_y=True)
|
|
46
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
47
|
+
X_train, X_val, y_train, y_val = _
|
|
48
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
49
|
+
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
50
|
+
assert 'forest' in perf.statistic()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_Perf_plot():
|
|
54
|
+
"""Test plot"""
|
|
55
|
+
|
|
56
|
+
from CompStats.interface import Perf
|
|
57
|
+
|
|
58
|
+
X, y = load_iris(return_X_y=True)
|
|
59
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
60
|
+
X_train, X_val, y_train, y_val = _
|
|
61
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
62
|
+
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
63
|
+
perf.plot()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_Perf_clone():
|
|
67
|
+
"""Test Perf.clone"""
|
|
68
|
+
from CompStats.interface import Perf
|
|
69
|
+
|
|
70
|
+
X, y = load_iris(return_X_y=True)
|
|
71
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
72
|
+
X_train, X_val, y_train, y_val = _
|
|
73
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
74
|
+
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
75
|
+
samples = perf.statistic_samples._samples
|
|
76
|
+
perf2 = clone(perf)
|
|
77
|
+
perf2.error_func = lambda y, hy: (y != hy).mean()
|
|
78
|
+
assert 'forest' in perf2.statistic_samples.calls
|
|
79
|
+
assert np.all(samples == perf2.statistic_samples._samples)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_Perf_best():
|
|
83
|
+
"""Test Perf.best"""
|
|
84
|
+
from CompStats.interface import Perf
|
|
85
|
+
|
|
86
|
+
X, y = load_iris(return_X_y=True)
|
|
87
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
88
|
+
X_train, X_val, y_train, y_val = _
|
|
89
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
90
|
+
hy = m.predict(X_val)
|
|
91
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
92
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
93
|
+
assert len(perf.best) == 2
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_Perf_difference():
|
|
97
|
+
"""Test difference"""
|
|
98
|
+
from CompStats.interface import Perf, Difference
|
|
99
|
+
|
|
100
|
+
X, y = load_iris(return_X_y=True)
|
|
101
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
102
|
+
X_train, X_val, y_train, y_val = _
|
|
103
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
104
|
+
hy = m.predict(X_val)
|
|
105
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
106
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
107
|
+
diff = perf.difference()
|
|
108
|
+
assert isinstance(diff, Difference)
|
|
109
|
+
assert isinstance(str(diff), str)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_Difference_plot():
|
|
113
|
+
"""Test difference plot"""
|
|
114
|
+
from CompStats.interface import Perf
|
|
115
|
+
|
|
116
|
+
X, y = load_iris(return_X_y=True)
|
|
117
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
118
|
+
X_train, X_val, y_train, y_val = _
|
|
119
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
120
|
+
hy = m.predict(X_val)
|
|
121
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
122
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
123
|
+
diff = perf.difference()
|
|
124
|
+
diff.plot()
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def test_Perf_dataframe():
|
|
128
|
+
"""Test Perf with dataframe"""
|
|
129
|
+
from CompStats.interface import Perf
|
|
130
|
+
|
|
131
|
+
df = pd.read_csv(DATA)
|
|
132
|
+
perf = Perf(df, num_samples=50)
|
|
133
|
+
assert 'INGEOTEC' in perf.statistic()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def test_Perf_call():
|
|
137
|
+
"""Test Perf call"""
|
|
138
|
+
from CompStats.interface import Perf
|
|
139
|
+
|
|
140
|
+
X, y = load_iris(return_X_y=True)
|
|
141
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
142
|
+
X_train, X_val, y_train, y_val = _
|
|
143
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
144
|
+
hy = m.predict(X_val)
|
|
145
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
146
|
+
hy2 = ens.predict(X_val)
|
|
147
|
+
perf = Perf(y_val, num_samples=50)
|
|
148
|
+
for xx in [hy, hy2]:
|
|
149
|
+
_ = perf(xx)
|
|
150
|
+
print(_)
|
|
151
|
+
perf(hy, name='alg-2')
|
|
152
|
+
assert 'alg-2' not in perf._statistic_samples.calls
|
|
153
|
+
assert 'alg-1' in perf._statistic_samples.calls
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
from sklearn.svm import LinearSVC
|
|
15
|
+
from sklearn.ensemble import RandomForestClassifier
|
|
16
|
+
from sklearn.datasets import load_iris
|
|
17
|
+
from sklearn.model_selection import train_test_split
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_f1_score():
|
|
21
|
+
"""Test f1_score"""
|
|
22
|
+
from CompStats.metrics import f1_score
|
|
23
|
+
|
|
24
|
+
X, y = load_iris(return_X_y=True)
|
|
25
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
26
|
+
X_train, X_val, y_train, y_val = _
|
|
27
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
28
|
+
perf = f1_score(y_val, forest=ens.predict(X_val),
|
|
29
|
+
num_samples=50, average='macro')
|
|
30
|
+
assert 'forest' in perf.statistic()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_accuracy_score():
|
|
34
|
+
"""Test f1_score"""
|
|
35
|
+
from CompStats.metrics import accuracy_score
|
|
36
|
+
|
|
37
|
+
X, y = load_iris(return_X_y=True)
|
|
38
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
39
|
+
X_train, X_val, y_train, y_val = _
|
|
40
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
41
|
+
perf = accuracy_score(y_val, forest=ens.predict(X_val),
|
|
42
|
+
num_samples=50)
|
|
43
|
+
assert 'forest' in perf.statistic()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_balanced_accuracy_score():
|
|
47
|
+
"""Test f1_score"""
|
|
48
|
+
from CompStats.metrics import balanced_accuracy_score
|
|
49
|
+
|
|
50
|
+
X, y = load_iris(return_X_y=True)
|
|
51
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
52
|
+
X_train, X_val, y_train, y_val = _
|
|
53
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
54
|
+
perf = balanced_accuracy_score(y_val, forest=ens.predict(X_val),
|
|
55
|
+
num_samples=50)
|
|
56
|
+
assert 'forest' in perf.statistic()
|
|
@@ -73,7 +73,7 @@ def test_performance_multiple_metrics():
|
|
|
73
73
|
{"func": mean_absolute_error, 'BiB': False}
|
|
74
74
|
]
|
|
75
75
|
perf = performance_multiple_metrics(df, "y", metrics)
|
|
76
|
-
|
|
76
|
+
plot_performance_multiple(perf)
|
|
77
77
|
assert 'accuracy_score' in perf['samples']
|
|
78
78
|
assert 'y' not in perf['samples']['accuracy_score']
|
|
79
79
|
assert 'INGEOTEC' in perf['samples']['accuracy_score']
|
|
@@ -90,7 +90,7 @@ def test_difference_multiple():
|
|
|
90
90
|
]
|
|
91
91
|
perf = performance_multiple_metrics(df, "y", metrics)
|
|
92
92
|
diff = difference_multiple(perf)
|
|
93
|
-
|
|
93
|
+
plot_difference_multiple(diff)
|
|
94
94
|
assert diff['winner']['accuracy_score']['best'] == 'BoW'
|
|
95
95
|
assert 'BoW' not in diff['winner']['accuracy_score']['diff'].keys()
|
|
96
96
|
# assert isinstance(ins, sns.FacetGrid)
|
|
@@ -18,8 +18,8 @@ except ImportError:
|
|
|
18
18
|
USE_TQDM = False
|
|
19
19
|
|
|
20
20
|
|
|
21
|
-
def progress_bar(arg, **kwargs):
|
|
21
|
+
def progress_bar(arg, use_tqdm: bool=True, **kwargs):
|
|
22
22
|
"""Progress bar using tqdm"""
|
|
23
|
-
if USE_TQDM:
|
|
24
|
-
return
|
|
25
|
-
return arg
|
|
23
|
+
if not USE_TQDM or not use_tqdm:
|
|
24
|
+
return arg
|
|
25
|
+
return tqdm(arg, **kwargs)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -5,7 +5,9 @@ pyproject.toml
|
|
|
5
5
|
setup.py
|
|
6
6
|
CompStats/__init__.py
|
|
7
7
|
CompStats/bootstrap.py
|
|
8
|
+
CompStats/interface.py
|
|
8
9
|
CompStats/measurements.py
|
|
10
|
+
CompStats/metrics.py
|
|
9
11
|
CompStats/performance.py
|
|
10
12
|
CompStats/utils.py
|
|
11
13
|
CompStats.egg-info/PKG-INFO
|
|
@@ -16,5 +18,7 @@ CompStats.egg-info/top_level.txt
|
|
|
16
18
|
CompStats/tests/__init__.py
|
|
17
19
|
CompStats/tests/data.csv
|
|
18
20
|
CompStats/tests/test_bootstrap.py
|
|
21
|
+
CompStats/tests/test_interface.py
|
|
19
22
|
CompStats/tests/test_measurements.py
|
|
23
|
+
CompStats/tests/test_metrics.py
|
|
20
24
|
CompStats/tests/test_performance.py
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|