CompStats 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
CompStats/__init__.py CHANGED
@@ -11,9 +11,10 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.2'
14
+ __version__ = '0.1.3'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
18
18
  from CompStats.performance import performance_multiple_metrics, difference_multiple, plot_performance_multiple, plot_difference_multiple
19
19
  from CompStats.performance import all_differences_multiple, plot_performance2, plot_difference2, plot_scatter_matrix
20
+ from CompStats.interface import Perf
CompStats/interface.py ADDED
@@ -0,0 +1,474 @@
1
+ # Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from dataclasses import dataclass
15
+ from sklearn.metrics import balanced_accuracy_score
16
+ from sklearn.base import clone
17
+ import pandas as pd
18
+ import numpy as np
19
+ from CompStats.bootstrap import StatisticSamples
20
+ from CompStats.utils import progress_bar
21
+ from CompStats.measurements import SE
22
+ from CompStats.performance import plot_performance, plot_difference
23
+
24
+
25
+ class Perf(object):
26
+ """Perf is an entry point to CompStats
27
+
28
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
29
+ :type y_true: numpy.ndarray or pandas.DataFrame
30
+ :param score_func: Function to measure the performance, it is assumed that the best algorithm has the highest value.
31
+ :type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
32
+ :param error_func: Function to measure the performance where the best algorithm has the lowest value.
33
+ :type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
34
+ :param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
35
+ :type args: numpy.ndarray
36
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
37
+ :type kwargs: numpy.ndarray
38
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
39
+ :type n_jobs: int
40
+ :param num_samples: Number of bootstrap samples, default=500.
41
+ :type num_samples: int
42
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True.
43
+ :type use_tqdm: bool
44
+
45
+
46
+ >>> from sklearn.svm import LinearSVC
47
+ >>> from sklearn.linear_model import LogisticRegression
48
+ >>> from sklearn.ensemble import RandomForestClassifier
49
+ >>> from sklearn.datasets import load_iris
50
+ >>> from sklearn.model_selection import train_test_split
51
+ >>> from sklearn.base import clone
52
+ >>> from CompStats.interface import Perf
53
+ >>> X, y = load_iris(return_X_y=True)
54
+ >>> _ = train_test_split(X, y, test_size=0.3)
55
+ >>> X_train, X_val, y_train, y_val = _
56
+ >>> m = LinearSVC().fit(X_train, y_train)
57
+ >>> hy = m.predict(X_val)
58
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
59
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
60
+ >>> perf
61
+ <Perf>
62
+ Prediction statistics with standard error
63
+ alg-1 = 1.000 (0.000)
64
+ forest = 0.946 (0.038)
65
+
66
+ If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
67
+
68
+ >>> lr = LogisticRegression().fit(X_train, y_train)
69
+ >>> perf(lr.predict(X_val), name='Log. Reg.')
70
+ <Perf>
71
+ Prediction statistics with standard error
72
+ alg-1 = 1.000 (0.000)
73
+ forest = 0.946 (0.038)
74
+ Log. Reg. = 0.946 (0.038)
75
+
76
+ The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
77
+
78
+ >>> perf_error = clone(perf)
79
+ >>> perf_error.error_func = lambda y, hy: (y != hy).mean()
80
+ >>> perf_error
81
+ <Perf>
82
+ Prediction statistics with standard error
83
+ alg-1 = 0.000 (0.000)
84
+ forest = 0.044 (0.030)
85
+ Log. Reg. = 0.044 (0.030)
86
+
87
+ """
88
+ def __init__(self, y_true, *args,
89
+ score_func=balanced_accuracy_score,
90
+ error_func=None,
91
+ num_samples: int=500,
92
+ n_jobs: int=-1,
93
+ use_tqdm=True,
94
+ **kwargs):
95
+ assert (score_func is None) ^ (error_func is None)
96
+ self.score_func = score_func
97
+ self.error_func = error_func
98
+ algs = {}
99
+ for k, v in enumerate(args):
100
+ algs[f'alg-{k+1}'] = np.asanyarray(v)
101
+ algs.update(**kwargs)
102
+ self.predictions = algs
103
+ self.y_true = y_true
104
+ self.num_samples = num_samples
105
+ self.n_jobs = n_jobs
106
+ self.use_tqdm = use_tqdm
107
+ self._init()
108
+
109
+ def _init(self):
110
+ """Compute the bootstrap statistic"""
111
+
112
+ bib = True if self.score_func is not None else False
113
+ if hasattr(self, '_statistic_samples'):
114
+ _ = self.statistic_samples
115
+ _.BiB = bib
116
+ else:
117
+ _ = StatisticSamples(statistic=self.statistic_func,
118
+ n_jobs=self.n_jobs,
119
+ num_samples=self.num_samples,
120
+ BiB=bib)
121
+ _.samples(N=self.y_true.shape[0])
122
+ self.statistic_samples = _
123
+
124
+ def get_params(self):
125
+ """Parameters"""
126
+
127
+ return dict(y_true=self.y_true,
128
+ score_func=self.score_func,
129
+ error_func=self.error_func,
130
+ num_samples=self.num_samples,
131
+ n_jobs=self.n_jobs)
132
+
133
+ def __sklearn_clone__(self):
134
+ klass = self.__class__
135
+ params = self.get_params()
136
+ ins = klass(**params)
137
+ ins.predictions = dict(self.predictions)
138
+ ins._statistic_samples._samples = self.statistic_samples._samples
139
+ return ins
140
+
141
+ def __repr__(self):
142
+ """Prediction statistics with standard error in parenthesis"""
143
+ return f"<{self.__class__.__name__}>\n{self}"
144
+
145
+ def __str__(self):
146
+ """Prediction statistics with standard error in parenthesis"""
147
+
148
+ se = self.se()
149
+ output = ["Prediction statistics with standard error"]
150
+ for key, value in self.statistic().items():
151
+ output.append(f'{key} = {value:0.3f} ({se[key]:0.3f})')
152
+ return "\n".join(output)
153
+
154
+ def __call__(self, y_pred, name=None):
155
+ """Add predictions"""
156
+ if name is None:
157
+ k = len(self.predictions) + 1
158
+ if k == 0:
159
+ k = 1
160
+ name = f'alg-{k}'
161
+ self.predictions[name] = np.asanyarray(y_pred)
162
+ samples = self._statistic_samples
163
+ calls = samples.calls
164
+ if name in calls:
165
+ del calls[name]
166
+ return self
167
+
168
+ def difference(self, wrt_to: str=None):
169
+ """Compute the difference w.r.t any algorithm by default is the best
170
+
171
+ >>> from sklearn.svm import LinearSVC
172
+ >>> from sklearn.ensemble import RandomForestClassifier
173
+ >>> from sklearn.datasets import load_iris
174
+ >>> from sklearn.model_selection import train_test_split
175
+ >>> from sklearn.base import clone
176
+ >>> from CompStats.interface import Perf
177
+ >>> X, y = load_iris(return_X_y=True)
178
+ >>> _ = train_test_split(X, y, test_size=0.3)
179
+ >>> X_train, X_val, y_train, y_val = _
180
+ >>> m = LinearSVC().fit(X_train, y_train)
181
+ >>> hy = m.predict(X_val)
182
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
183
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
184
+ >>> perf.difference()
185
+ <Difference>
186
+ difference p-values w.r.t alg-1
187
+ forest 0.06
188
+ """
189
+ if wrt_to is None:
190
+ wrt_to = self.best[0]
191
+ base = self.statistic_samples.calls[wrt_to]
192
+ sign = 1 if self.statistic_samples.BiB else -1
193
+ diff = dict()
194
+ for k, v in self.statistic_samples.calls.items():
195
+ if k == wrt_to:
196
+ continue
197
+ diff[k] = sign * (base - v)
198
+ diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
199
+ statistic=self.statistic(),
200
+ best=self.best[0])
201
+ diff_ins.statistic_samples.calls = diff
202
+ diff_ins.statistic_samples.info['best'] = self.best[0]
203
+ return diff_ins
204
+
205
+ @property
206
+ def best(self):
207
+ """Best system"""
208
+
209
+ try:
210
+ return self._best
211
+ except AttributeError:
212
+ statistic = [(k, v) for k, v in self.statistic().items()]
213
+ statistic = sorted(statistic, key=lambda x: x[1],
214
+ reverse=self.statistic_samples.BiB)
215
+ self._best = statistic[0]
216
+ return self._best
217
+
218
+ def statistic(self):
219
+ """Statistic
220
+
221
+ >>> from sklearn.svm import LinearSVC
222
+ >>> from sklearn.ensemble import RandomForestClassifier
223
+ >>> from sklearn.datasets import load_iris
224
+ >>> from sklearn.model_selection import train_test_split
225
+ >>> from CompStats.interface import Perf
226
+ >>> X, y = load_iris(return_X_y=True)
227
+ >>> _ = train_test_split(X, y, test_size=0.3)
228
+ >>> X_train, X_val, y_train, y_val = _
229
+ >>> m = LinearSVC().fit(X_train, y_train)
230
+ >>> hy = m.predict(X_val)
231
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
232
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
233
+ >>> perf.statistic()
234
+ {'alg-1': 1.0, 'forest': 0.9500891265597148}
235
+ """
236
+
237
+ data = sorted([(k, self.statistic_func(self.y_true, v))
238
+ for k, v in self.predictions.items()],
239
+ key=lambda x: x[1], reverse=self.statistic_samples.BiB)
240
+ return dict(data)
241
+
242
+ def se(self):
243
+ """Standard Error
244
+
245
+ >>> from sklearn.svm import LinearSVC
246
+ >>> from sklearn.ensemble import RandomForestClassifier
247
+ >>> from sklearn.datasets import load_iris
248
+ >>> from sklearn.model_selection import train_test_split
249
+ >>> from CompStats.interface import Perf
250
+ >>> X, y = load_iris(return_X_y=True)
251
+ >>> _ = train_test_split(X, y, test_size=0.3)
252
+ >>> X_train, X_val, y_train, y_val = _
253
+ >>> m = LinearSVC().fit(X_train, y_train)
254
+ >>> hy = m.predict(X_val)
255
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
256
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
257
+ >>> print(perf.se())
258
+ {'alg-1': 0.0, 'forest': 0.026945730782184187}
259
+ """
260
+ return SE(self.statistic_samples)
261
+
262
+ def plot(self, **kwargs):
263
+ """plot with seaborn
264
+
265
+ >>> from sklearn.svm import LinearSVC
266
+ >>> from sklearn.ensemble import RandomForestClassifier
267
+ >>> from sklearn.datasets import load_iris
268
+ >>> from sklearn.model_selection import train_test_split
269
+ >>> from CompStats.interface import Perf
270
+ >>> X, y = load_iris(return_X_y=True)
271
+ >>> _ = train_test_split(X, y, test_size=0.3)
272
+ >>> X_train, X_val, y_train, y_val = _
273
+ >>> m = LinearSVC().fit(X_train, y_train)
274
+ >>> hy = m.predict(X_val)
275
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
276
+ >>> perf = Perf(y_val, hy, score_func=None,
277
+ error_func=lambda y, hy: (y != hy).mean(),
278
+ forest=ens.predict(X_val))
279
+ >>> perf.plot()
280
+ """
281
+ if self.score_func is not None:
282
+ value_name = 'Score'
283
+ else:
284
+ value_name = 'Error'
285
+ _ = dict(value_name=value_name)
286
+ _.update(kwargs)
287
+ return plot_performance(self.statistic_samples, **_)
288
+
289
+ @property
290
+ def n_jobs(self):
291
+ """Number of jobs to compute the statistics"""
292
+ return self._n_jobs
293
+
294
+ @n_jobs.setter
295
+ def n_jobs(self, value):
296
+ self._n_jobs = value
297
+
298
+ @property
299
+ def statistic_func(self):
300
+ """Statistic function"""
301
+ if self.score_func is not None:
302
+ return self.score_func
303
+ return self.error_func
304
+
305
+ @property
306
+ def statistic_samples(self):
307
+ """Statistic Samples"""
308
+
309
+ samples = self._statistic_samples
310
+ algs = set(samples.calls.keys())
311
+ algs = set(self.predictions.keys()) - algs
312
+ if len(algs):
313
+ for key in progress_bar(algs, use_tqdm=self.use_tqdm):
314
+ samples(self.y_true, self.predictions[key], name=key)
315
+ return self._statistic_samples
316
+
317
+ @statistic_samples.setter
318
+ def statistic_samples(self, value):
319
+ self._statistic_samples = value
320
+
321
+ @property
322
+ def num_samples(self):
323
+ """Number of bootstrap samples"""
324
+ return self._num_samples
325
+
326
+ @num_samples.setter
327
+ def num_samples(self, value):
328
+ self._num_samples = value
329
+
330
+ @property
331
+ def predictions(self):
332
+ """Predictions"""
333
+ return self._predictions
334
+
335
+ @predictions.setter
336
+ def predictions(self, value):
337
+ self._predictions = value
338
+
339
+ @property
340
+ def y_true(self):
341
+ """True output, gold standard o :math:`y`"""
342
+
343
+ return self._y_true
344
+
345
+ @y_true.setter
346
+ def y_true(self, value):
347
+ if isinstance(value, pd.DataFrame):
348
+ self._y_true = value['y'].to_numpy()
349
+ algs = {}
350
+ for c in value.columns:
351
+ if c == 'y':
352
+ continue
353
+ algs[c] = value[c].to_numpy()
354
+ self.predictions.update(algs)
355
+ return
356
+ self._y_true = value
357
+
358
+ @property
359
+ def score_func(self):
360
+ """Score function"""
361
+ return self._score_func
362
+
363
+ @score_func.setter
364
+ def score_func(self, value):
365
+ self._score_func = value
366
+ if value is not None:
367
+ self.error_func = None
368
+ if hasattr(self, '_statistic_samples'):
369
+ self._statistic_samples.statistic = value
370
+ self._statistic_samples.BiB = True
371
+
372
+ @property
373
+ def error_func(self):
374
+ """Error function"""
375
+ return self._error_func
376
+
377
+ @error_func.setter
378
+ def error_func(self, value):
379
+ self._error_func = value
380
+ if value is not None:
381
+ self.score_func = None
382
+ if hasattr(self, '_statistic_samples'):
383
+ self._statistic_samples.statistic = value
384
+ self._statistic_samples.BiB = False
385
+
386
+
387
+ @dataclass
388
+ class Difference:
389
+ """Difference
390
+
391
+ >>> from sklearn.svm import LinearSVC
392
+ >>> from sklearn.ensemble import RandomForestClassifier
393
+ >>> from sklearn.datasets import load_iris
394
+ >>> from sklearn.model_selection import train_test_split
395
+ >>> from sklearn.base import clone
396
+ >>> from CompStats.interface import Perf
397
+ >>> X, y = load_iris(return_X_y=True)
398
+ >>> _ = train_test_split(X, y, test_size=0.3)
399
+ >>> X_train, X_val, y_train, y_val = _
400
+ >>> m = LinearSVC().fit(X_train, y_train)
401
+ >>> hy = m.predict(X_val)
402
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
403
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
404
+ >>> diff = perf.difference()
405
+ >>> diff
406
+ <Difference>
407
+ difference p-values w.r.t alg-1
408
+ forest 0.3
409
+ """
410
+
411
+ statistic_samples:StatisticSamples=None
412
+ best:str=None
413
+ statistic:dict=None
414
+
415
+ def __repr__(self):
416
+ """p-value"""
417
+ return f"<{self.__class__.__name__}>\n{self}"
418
+
419
+ def __str__(self):
420
+ """p-value"""
421
+ output = [f"difference p-values w.r.t {self.best}"]
422
+ for k, v in self.p_value().items():
423
+ output.append(f'{k} {v}')
424
+ return "\n".join(output)
425
+
426
+ def p_value(self):
427
+ """Compute p_value of the differences
428
+
429
+ >>> from sklearn.svm import LinearSVC
430
+ >>> from sklearn.ensemble import RandomForestClassifier
431
+ >>> from sklearn.datasets import load_iris
432
+ >>> from sklearn.model_selection import train_test_split
433
+ >>> from sklearn.base import clone
434
+ >>> from CompStats.interface import Perf
435
+ >>> X, y = load_iris(return_X_y=True)
436
+ >>> _ = train_test_split(X, y, test_size=0.3)
437
+ >>> X_train, X_val, y_train, y_val = _
438
+ >>> m = LinearSVC().fit(X_train, y_train)
439
+ >>> hy = m.predict(X_val)
440
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
441
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
442
+ >>> diff = perf.difference()
443
+ >>> diff.p_value()
444
+ {'forest': np.float64(0.3)}
445
+ """
446
+ values = []
447
+ sign = 1 if self.statistic_samples.BiB else -1
448
+ for k, v in self.statistic_samples.calls.items():
449
+ delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
450
+ values.append((k, (v > delta).mean()))
451
+ values.sort(key=lambda x: x[1])
452
+ return dict(values)
453
+
454
+ def plot(self, **kwargs):
455
+ """Plot
456
+
457
+ >>> from sklearn.svm import LinearSVC
458
+ >>> from sklearn.ensemble import RandomForestClassifier
459
+ >>> from sklearn.datasets import load_iris
460
+ >>> from sklearn.model_selection import train_test_split
461
+ >>> from sklearn.base import clone
462
+ >>> from CompStats.interface import Perf
463
+ >>> X, y = load_iris(return_X_y=True)
464
+ >>> _ = train_test_split(X, y, test_size=0.3)
465
+ >>> X_train, X_val, y_train, y_val = _
466
+ >>> m = LinearSVC().fit(X_train, y_train)
467
+ >>> hy = m.predict(X_val)
468
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
469
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
470
+ >>> diff = perf.difference()
471
+ >>> diff.plot()
472
+ """
473
+
474
+ return plot_difference(self.statistic_samples, **kwargs)
CompStats/metrics.py ADDED
@@ -0,0 +1,197 @@
1
+ # Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from CompStats.interface import Perf
15
+ from sklearn import metrics
16
+
17
+
18
+ def accuracy_score(y_true, *args,
19
+ normalize=True, sample_weight=None,
20
+ num_samples: int=500,
21
+ n_jobs: int=-1,
22
+ use_tqdm=True,
23
+ **kwargs):
24
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`sklearn.metrics.accuracy_score` as :py:attr:`score_func.`
25
+
26
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
27
+ :type y_true: numpy.ndarray or pandas.DataFrame
28
+ :param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
29
+ :type args: numpy.ndarray
30
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
31
+ :type args: numpy.ndarray
32
+ :param num_samples: Number of bootstrap samples, default=500.
33
+ :type num_samples: int
34
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
35
+ :type n_jobs: int
36
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
37
+ :type use_tqdm: bool
38
+ :param normalize: see sklearn.metrics.f1_score
39
+ :param sample_weight: see sklearn.metrics.f1_score
40
+
41
+ >>> from sklearn.svm import LinearSVC
42
+ >>> from sklearn.ensemble import RandomForestClassifier
43
+ >>> from sklearn.datasets import load_iris
44
+ >>> from sklearn.model_selection import train_test_split
45
+ >>> from sklearn.base import clone
46
+ >>> from CompStats.metrics import accuracy_score
47
+ >>> X, y = load_iris(return_X_y=True)
48
+ >>> _ = train_test_split(X, y, test_size=0.3)
49
+ >>> X_train, X_val, y_train, y_val = _
50
+ >>> m = LinearSVC().fit(X_train, y_train)
51
+ >>> hy = m.predict(X_val)
52
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
53
+ >>> score = accuracy_score(y_val, hy,
54
+ forest=ens.predict(X_val))
55
+ >>> score
56
+ <Perf>
57
+ Prediction statistics with standard error
58
+ forest = 0.978 (0.023)
59
+ alg-1 = 0.956 (0.030)
60
+ >>> diff = score.difference()
61
+ >>> diff
62
+ <Difference>
63
+ difference p-values w.r.t forest
64
+ alg-1 0.252
65
+ """
66
+
67
+ def inner(y, hy):
68
+ return metrics.accuracy_score(y, hy,
69
+ normalize=normalize,
70
+ sample_weight=sample_weight)
71
+ return Perf(y_true, *args, score_func=inner,
72
+ num_samples=num_samples, n_jobs=n_jobs,
73
+ use_tqdm=use_tqdm,
74
+ **kwargs)
75
+
76
+
77
+ def balanced_accuracy_score(y_true, *args,
78
+ sample_weight=None, adjusted=False,
79
+ num_samples: int=500,
80
+ n_jobs: int=-1,
81
+ use_tqdm=True,
82
+ **kwargs):
83
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`sklearn.metrics.balanced_accuracy_score` as :py:attr:`score_func.`
84
+
85
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
86
+ :type y_true: numpy.ndarray or pandas.DataFrame
87
+ :param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
88
+ :type args: numpy.ndarray
89
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
90
+ :type args: numpy.ndarray
91
+ :param num_samples: Number of bootstrap samples, default=500.
92
+ :type num_samples: int
93
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
94
+ :type n_jobs: int
95
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
96
+ :type use_tqdm: bool
97
+ :param sample_weight: see sklearn.metrics.f1_score
98
+ :param adjusted: see sklearn.metrics.f1_score
99
+
100
+ >>> from sklearn.svm import LinearSVC
101
+ >>> from sklearn.ensemble import RandomForestClassifier
102
+ >>> from sklearn.datasets import load_iris
103
+ >>> from sklearn.model_selection import train_test_split
104
+ >>> from sklearn.base import clone
105
+ >>> from CompStats.metrics import balanced_accuracy_score
106
+ >>> X, y = load_iris(return_X_y=True)
107
+ >>> _ = train_test_split(X, y, test_size=0.3)
108
+ >>> X_train, X_val, y_train, y_val = _
109
+ >>> m = LinearSVC().fit(X_train, y_train)
110
+ >>> hy = m.predict(X_val)
111
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
112
+ >>> score = balanced_accuracy_score(y_val, hy,
113
+ forest=ens.predict(X_val))
114
+ >>> score
115
+ <Perf>
116
+ Prediction statistics with standard error
117
+ forest = 0.957 (0.031)
118
+ alg-1 = 0.935 (0.037)
119
+ >>> diff = score.difference()
120
+ >>> diff
121
+ <Difference>
122
+ difference p-values w.r.t forest
123
+ alg-1 0.254
124
+ """
125
+
126
+ def inner(y, hy):
127
+ return metrics.balanced_accuracy_score(y, hy,
128
+ adjusted=adjusted,
129
+ sample_weight=sample_weight)
130
+ return Perf(y_true, *args, score_func=inner,
131
+ num_samples=num_samples, n_jobs=n_jobs,
132
+ use_tqdm=use_tqdm,
133
+ **kwargs)
134
+
135
+
136
+ def f1_score(y_true, *args, labels=None, pos_label=1,
137
+ average='binary', sample_weight=None,
138
+ zero_division='warn', num_samples: int=500,
139
+ n_jobs: int=-1, use_tqdm=True,
140
+ **kwargs):
141
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`sklearn.metrics.f1_score` as :py:attr:`score_func.`
142
+
143
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
144
+ :type y_true: numpy.ndarray or pandas.DataFrame
145
+ :param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
146
+ :type args: numpy.ndarray
147
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
148
+ :type args: numpy.ndarray
149
+ :param num_samples: Number of bootstrap samples, default=500.
150
+ :type num_samples: int
151
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
152
+ :type n_jobs: int
153
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
154
+ :type use_tqdm: bool
155
+ :param labels: see sklearn.metrics.f1_score
156
+ :param pos_label: see sklearn.metrics.f1_score
157
+ :param average: see sklearn.metrics.f1_score
158
+ :param sample_weight: see sklearn.metrics.f1_score
159
+ :param zero_division: see sklearn.metrics.f1_score
160
+
161
+ >>> from sklearn.svm import LinearSVC
162
+ >>> from sklearn.ensemble import RandomForestClassifier
163
+ >>> from sklearn.datasets import load_iris
164
+ >>> from sklearn.model_selection import train_test_split
165
+ >>> from sklearn.base import clone
166
+ >>> from CompStats.metrics import f1_score
167
+ >>> X, y = load_iris(return_X_y=True)
168
+ >>> _ = train_test_split(X, y, test_size=0.3)
169
+ >>> X_train, X_val, y_train, y_val = _
170
+ >>> m = LinearSVC().fit(X_train, y_train)
171
+ >>> hy = m.predict(X_val)
172
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
173
+ >>> score = f1_score(y_val, hy,
174
+ forest=ens.predict(X_val),
175
+ average='macro')
176
+ >>> score
177
+ <Perf>
178
+ Prediction statistics with standard error
179
+ forest = 0.954 (0.032)
180
+ alg-1 = 0.931 (0.040)
181
+ >>> diff = score.difference()
182
+ >>> diff
183
+ <Difference>
184
+ difference p-values w.r.t forest
185
+ alg-1 0.176
186
+ """
187
+
188
+ def inner(y, hy):
189
+ return metrics.f1_score(y, hy, labels=labels,
190
+ pos_label=pos_label,
191
+ average=average,
192
+ sample_weight=sample_weight,
193
+ zero_division=zero_division)
194
+ return Perf(y_true, *args, score_func=inner,
195
+ num_samples=num_samples, n_jobs=n_jobs,
196
+ use_tqdm=use_tqdm,
197
+ **kwargs)
@@ -0,0 +1,153 @@
1
+ # Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import numpy as np
16
+ from sklearn.base import clone
17
+ from sklearn.svm import LinearSVC
18
+ from sklearn.ensemble import RandomForestClassifier
19
+ from sklearn.datasets import load_iris
20
+ from sklearn.model_selection import train_test_split
21
+ import pandas as pd
22
+ from CompStats.tests.test_performance import DATA
23
+
24
+
25
+ def test_Perf():
26
+ """Test perf"""
27
+ from CompStats.interface import Perf
28
+
29
+ X, y = load_iris(return_X_y=True)
30
+ _ = train_test_split(X, y, test_size=0.3)
31
+ X_train, X_val, y_train, y_val = _
32
+ m = LinearSVC().fit(X_train, y_train)
33
+ hy = m.predict(X_val)
34
+ ens = RandomForestClassifier().fit(X_train, y_train)
35
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
36
+ assert 'alg-1' in perf.predictions
37
+ assert 'forest' in perf.predictions
38
+ assert str(perf) is not None
39
+
40
+
41
+ def test_Perf_statistic():
42
+ """Test statistic"""
43
+ from CompStats.interface import Perf
44
+
45
+ X, y = load_iris(return_X_y=True)
46
+ _ = train_test_split(X, y, test_size=0.3)
47
+ X_train, X_val, y_train, y_val = _
48
+ ens = RandomForestClassifier().fit(X_train, y_train)
49
+ perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
50
+ assert 'forest' in perf.statistic()
51
+
52
+
53
+ def test_Perf_plot():
54
+ """Test plot"""
55
+
56
+ from CompStats.interface import Perf
57
+
58
+ X, y = load_iris(return_X_y=True)
59
+ _ = train_test_split(X, y, test_size=0.3)
60
+ X_train, X_val, y_train, y_val = _
61
+ ens = RandomForestClassifier().fit(X_train, y_train)
62
+ perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
63
+ perf.plot()
64
+
65
+
66
+ def test_Perf_clone():
67
+ """Test Perf.clone"""
68
+ from CompStats.interface import Perf
69
+
70
+ X, y = load_iris(return_X_y=True)
71
+ _ = train_test_split(X, y, test_size=0.3)
72
+ X_train, X_val, y_train, y_val = _
73
+ ens = RandomForestClassifier().fit(X_train, y_train)
74
+ perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
75
+ samples = perf.statistic_samples._samples
76
+ perf2 = clone(perf)
77
+ perf2.error_func = lambda y, hy: (y != hy).mean()
78
+ assert 'forest' in perf2.statistic_samples.calls
79
+ assert np.all(samples == perf2.statistic_samples._samples)
80
+
81
+
82
+ def test_Perf_best():
83
+ """Test Perf.best"""
84
+ from CompStats.interface import Perf
85
+
86
+ X, y = load_iris(return_X_y=True)
87
+ _ = train_test_split(X, y, test_size=0.3)
88
+ X_train, X_val, y_train, y_val = _
89
+ m = LinearSVC().fit(X_train, y_train)
90
+ hy = m.predict(X_val)
91
+ ens = RandomForestClassifier().fit(X_train, y_train)
92
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
93
+ assert len(perf.best) == 2
94
+
95
+
96
+ def test_Perf_difference():
97
+ """Test difference"""
98
+ from CompStats.interface import Perf, Difference
99
+
100
+ X, y = load_iris(return_X_y=True)
101
+ _ = train_test_split(X, y, test_size=0.3)
102
+ X_train, X_val, y_train, y_val = _
103
+ m = LinearSVC().fit(X_train, y_train)
104
+ hy = m.predict(X_val)
105
+ ens = RandomForestClassifier().fit(X_train, y_train)
106
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
107
+ diff = perf.difference()
108
+ assert isinstance(diff, Difference)
109
+ assert isinstance(str(diff), str)
110
+
111
+
112
+ def test_Difference_plot():
113
+ """Test difference plot"""
114
+ from CompStats.interface import Perf
115
+
116
+ X, y = load_iris(return_X_y=True)
117
+ _ = train_test_split(X, y, test_size=0.3)
118
+ X_train, X_val, y_train, y_val = _
119
+ m = LinearSVC().fit(X_train, y_train)
120
+ hy = m.predict(X_val)
121
+ ens = RandomForestClassifier().fit(X_train, y_train)
122
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
123
+ diff = perf.difference()
124
+ diff.plot()
125
+
126
+
127
+ def test_Perf_dataframe():
128
+ """Test Perf with dataframe"""
129
+ from CompStats.interface import Perf
130
+
131
+ df = pd.read_csv(DATA)
132
+ perf = Perf(df, num_samples=50)
133
+ assert 'INGEOTEC' in perf.statistic()
134
+
135
+
136
+ def test_Perf_call():
137
+ """Test Perf call"""
138
+ from CompStats.interface import Perf
139
+
140
+ X, y = load_iris(return_X_y=True)
141
+ _ = train_test_split(X, y, test_size=0.3)
142
+ X_train, X_val, y_train, y_val = _
143
+ m = LinearSVC().fit(X_train, y_train)
144
+ hy = m.predict(X_val)
145
+ ens = RandomForestClassifier().fit(X_train, y_train)
146
+ hy2 = ens.predict(X_val)
147
+ perf = Perf(y_val, num_samples=50)
148
+ for xx in [hy, hy2]:
149
+ _ = perf(xx)
150
+ print(_)
151
+ perf(hy, name='alg-2')
152
+ assert 'alg-2' not in perf._statistic_samples.calls
153
+ assert 'alg-1' in perf._statistic_samples.calls
@@ -0,0 +1,56 @@
1
+ # Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from sklearn.svm import LinearSVC
15
+ from sklearn.ensemble import RandomForestClassifier
16
+ from sklearn.datasets import load_iris
17
+ from sklearn.model_selection import train_test_split
18
+
19
+
20
+ def test_f1_score():
21
+ """Test f1_score"""
22
+ from CompStats.metrics import f1_score
23
+
24
+ X, y = load_iris(return_X_y=True)
25
+ _ = train_test_split(X, y, test_size=0.3)
26
+ X_train, X_val, y_train, y_val = _
27
+ ens = RandomForestClassifier().fit(X_train, y_train)
28
+ perf = f1_score(y_val, forest=ens.predict(X_val),
29
+ num_samples=50, average='macro')
30
+ assert 'forest' in perf.statistic()
31
+
32
+
33
+ def test_accuracy_score():
34
+ """Test f1_score"""
35
+ from CompStats.metrics import accuracy_score
36
+
37
+ X, y = load_iris(return_X_y=True)
38
+ _ = train_test_split(X, y, test_size=0.3)
39
+ X_train, X_val, y_train, y_val = _
40
+ ens = RandomForestClassifier().fit(X_train, y_train)
41
+ perf = accuracy_score(y_val, forest=ens.predict(X_val),
42
+ num_samples=50)
43
+ assert 'forest' in perf.statistic()
44
+
45
+
46
+ def test_balanced_accuracy_score():
47
+ """Test f1_score"""
48
+ from CompStats.metrics import balanced_accuracy_score
49
+
50
+ X, y = load_iris(return_X_y=True)
51
+ _ = train_test_split(X, y, test_size=0.3)
52
+ X_train, X_val, y_train, y_val = _
53
+ ens = RandomForestClassifier().fit(X_train, y_train)
54
+ perf = balanced_accuracy_score(y_val, forest=ens.predict(X_val),
55
+ num_samples=50)
56
+ assert 'forest' in perf.statistic()
@@ -73,7 +73,7 @@ def test_performance_multiple_metrics():
73
73
  {"func": mean_absolute_error, 'BiB': False}
74
74
  ]
75
75
  perf = performance_multiple_metrics(df, "y", metrics)
76
- ins = plot_performance_multiple(perf)
76
+ plot_performance_multiple(perf)
77
77
  assert 'accuracy_score' in perf['samples']
78
78
  assert 'y' not in perf['samples']['accuracy_score']
79
79
  assert 'INGEOTEC' in perf['samples']['accuracy_score']
@@ -90,7 +90,7 @@ def test_difference_multiple():
90
90
  ]
91
91
  perf = performance_multiple_metrics(df, "y", metrics)
92
92
  diff = difference_multiple(perf)
93
- ins = plot_difference_multiple(diff)
93
+ plot_difference_multiple(diff)
94
94
  assert diff['winner']['accuracy_score']['best'] == 'BoW'
95
95
  assert 'BoW' not in diff['winner']['accuracy_score']['diff'].keys()
96
96
  # assert isinstance(ins, sns.FacetGrid)
CompStats/utils.py CHANGED
@@ -18,8 +18,8 @@ except ImportError:
18
18
  USE_TQDM = False
19
19
 
20
20
 
21
- def progress_bar(arg, **kwargs):
21
+ def progress_bar(arg, use_tqdm: bool=True, **kwargs):
22
22
  """Progress bar using tqdm"""
23
- if USE_TQDM:
24
- return tqdm(arg, **kwargs)
25
- return arg
23
+ if not USE_TQDM or not use_tqdm:
24
+ return arg
25
+ return tqdm(arg, **kwargs)
@@ -1,6 +1,6 @@
1
- Metadata-Version: 2.1
1
+ Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.2
3
+ Version: 0.1.3
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -18,9 +18,9 @@ Classifier: Topic :: Scientific/Engineering :: Information Analysis
18
18
  Description-Content-Type: text/x-rst
19
19
  License-File: LICENSE
20
20
  Requires-Dist: numpy
21
- Requires-Dist: scikit-learn >=1.3.0
21
+ Requires-Dist: scikit-learn>=1.3.0
22
22
  Requires-Dist: pandas
23
- Requires-Dist: seaborn >=0.13.0
23
+ Requires-Dist: seaborn>=0.13.0
24
24
 
25
25
  ====================================
26
26
  CompStats
@@ -0,0 +1,19 @@
1
+ CompStats/__init__.py,sha256=keRrVCM5OQjNPGm_dp6MszITQ3wj4c65m8AudDQJnMw,1156
2
+ CompStats/bootstrap.py,sha256=luI8iZkYTIDxnUaPb6zuGrVnsS5mrSXbbC7LCEk4s4A,10639
3
+ CompStats/interface.py,sha256=ioYTo1L9RoXKZn7x5DGnJ14VHYiVAiidG7T6uIm4msA,18096
4
+ CompStats/measurements.py,sha256=W7MupdEdluZ3P69CY6upvh8B2ilvyKQkguhdwbzGnQo,2867
5
+ CompStats/metrics.py,sha256=J1b5sYJJek6-dcJBeWR7vhm4pfxhU5J1FLOgQc2bJ7I,8871
6
+ CompStats/performance.py,sha256=wPLsyUmg3JBCYWZ8CTahr3JB5luBwqxCiYO1QCSYsIo,33710
7
+ CompStats/utils.py,sha256=gvp2R8cwXwLpoNyIrhati6LSLVbp_6K5opIXlTZMg04,898
8
+ CompStats/tests/__init__.py,sha256=KkM3LFXv7SODRJwWf6tl1kexygR-Jjcgedkdm8Rb5mo,615
9
+ CompStats/tests/data.csv,sha256=-t4ABChipLFhYn6pNZfHIoc4E3XEbamP03BryvGV88s,23763
10
+ CompStats/tests/test_bootstrap.py,sha256=QMXe8VB__QkSTFqDdKRb9Z3NnM58qBoO97dYCkd4G94,3888
11
+ CompStats/tests/test_interface.py,sha256=2XUzbqeP0b0xEYNXvcaTTQrjISJ6PABRE_0yJ7U0_R4,5132
12
+ CompStats/tests/test_measurements.py,sha256=t-gOhUE5AkBu5Dt6G0ot8ihdrRX-jH-nkGJYycdE-Ac,1758
13
+ CompStats/tests/test_metrics.py,sha256=vQZQBixqNSgUcO_Lj6DDKqwSfMyW1S2sjCpx45daugA,2148
14
+ CompStats/tests/test_performance.py,sha256=0cfJnFIe2yE5TeZ59kkjVR2S1nLk78J2m_yCK4Knu90,4456
15
+ CompStats-0.1.3.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
16
+ CompStats-0.1.3.dist-info/METADATA,sha256=XTK4UpqQ7V_ZEBZRBOLIImFBNevZH4_lbON3U6gu90Y,3301
17
+ CompStats-0.1.3.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
18
+ CompStats-0.1.3.dist-info/top_level.txt,sha256=dK1wZh1NWAXl3-aLQCSp1ds4FsuxZrmwV0IPnP9fzWM,10
19
+ CompStats-0.1.3.dist-info/RECORD,,
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (75.2.0)
2
+ Generator: setuptools (75.8.0)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5
 
@@ -1,15 +0,0 @@
1
- CompStats/__init__.py,sha256=8WeegLFroFjRjeln7HkxUjGIOexjUoVQ7q4rn-aO25M,1120
2
- CompStats/bootstrap.py,sha256=luI8iZkYTIDxnUaPb6zuGrVnsS5mrSXbbC7LCEk4s4A,10639
3
- CompStats/measurements.py,sha256=W7MupdEdluZ3P69CY6upvh8B2ilvyKQkguhdwbzGnQo,2867
4
- CompStats/performance.py,sha256=wPLsyUmg3JBCYWZ8CTahr3JB5luBwqxCiYO1QCSYsIo,33710
5
- CompStats/utils.py,sha256=7eRI53ZCqOTV_e4VAX0tYfQQPLm7BPP13rLZ2fZ-cdw,857
6
- CompStats/tests/__init__.py,sha256=KkM3LFXv7SODRJwWf6tl1kexygR-Jjcgedkdm8Rb5mo,615
7
- CompStats/tests/data.csv,sha256=-t4ABChipLFhYn6pNZfHIoc4E3XEbamP03BryvGV88s,23763
8
- CompStats/tests/test_bootstrap.py,sha256=QMXe8VB__QkSTFqDdKRb9Z3NnM58qBoO97dYCkd4G94,3888
9
- CompStats/tests/test_measurements.py,sha256=t-gOhUE5AkBu5Dt6G0ot8ihdrRX-jH-nkGJYycdE-Ac,1758
10
- CompStats/tests/test_performance.py,sha256=o3WFCKOT_txdRv_y488tZKH7o07jQRCT1mjLWJtX9pQ,4468
11
- CompStats-0.1.2.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
12
- CompStats-0.1.2.dist-info/METADATA,sha256=u-YtLKQmU4qMDi6_Ew5tr94gdZMYC6wVsR2LapItS9U,3303
13
- CompStats-0.1.2.dist-info/WHEEL,sha256=OVMc5UfuAQiSplgO0_WdW7vXVGAt9Hdd6qtN4HotdyA,91
14
- CompStats-0.1.2.dist-info/top_level.txt,sha256=dK1wZh1NWAXl3-aLQCSp1ds4FsuxZrmwV0IPnP9fzWM,10
15
- CompStats-0.1.2.dist-info/RECORD,,