CompStats 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
CompStats/__init__.py CHANGED
@@ -11,9 +11,10 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.2'
14
+ __version__ = '0.1.4'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
18
18
  from CompStats.performance import performance_multiple_metrics, difference_multiple, plot_performance_multiple, plot_difference_multiple
19
19
  from CompStats.performance import all_differences_multiple, plot_performance2, plot_difference2, plot_scatter_matrix
20
+ from CompStats.interface import Perf
CompStats/interface.py ADDED
@@ -0,0 +1,481 @@
1
+ # Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from dataclasses import dataclass
15
+ from sklearn.metrics import balanced_accuracy_score
16
+ from sklearn.base import clone
17
+ import pandas as pd
18
+ import numpy as np
19
+ from CompStats.bootstrap import StatisticSamples
20
+ from CompStats.utils import progress_bar
21
+ from CompStats.measurements import SE
22
+ from CompStats.performance import plot_performance, plot_difference
23
+
24
+
25
+ class Perf(object):
26
+ """Perf is an entry point to CompStats
27
+
28
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
29
+ :type y_true: numpy.ndarray or pandas.DataFrame
30
+ :param score_func: Function to measure the performance, it is assumed that the best algorithm has the highest value.
31
+ :type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
32
+ :param error_func: Function to measure the performance where the best algorithm has the lowest value.
33
+ :type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
34
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
35
+ :type y_pred: numpy.ndarray
36
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
37
+ :type kwargs: numpy.ndarray
38
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
39
+ :type n_jobs: int
40
+ :param num_samples: Number of bootstrap samples, default=500.
41
+ :type num_samples: int
42
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True.
43
+ :type use_tqdm: bool
44
+
45
+
46
+ >>> from sklearn.svm import LinearSVC
47
+ >>> from sklearn.linear_model import LogisticRegression
48
+ >>> from sklearn.ensemble import RandomForestClassifier
49
+ >>> from sklearn.datasets import load_iris
50
+ >>> from sklearn.model_selection import train_test_split
51
+ >>> from sklearn.base import clone
52
+ >>> from CompStats.interface import Perf
53
+ >>> X, y = load_iris(return_X_y=True)
54
+ >>> _ = train_test_split(X, y, test_size=0.3)
55
+ >>> X_train, X_val, y_train, y_val = _
56
+ >>> m = LinearSVC().fit(X_train, y_train)
57
+ >>> hy = m.predict(X_val)
58
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
59
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
60
+ >>> perf
61
+ <Perf>
62
+ Statistic with its standard error (se)
63
+ statistic (se)
64
+ 0.9792 (0.0221) <= alg-1
65
+ 0.9744 (0.0246) <= forest
66
+
67
+ If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
68
+
69
+ >>> lr = LogisticRegression().fit(X_train, y_train)
70
+ >>> perf(lr.predict(X_val), name='Log. Reg.')
71
+ <Perf>
72
+ Statistic with its standard error (se)
73
+ statistic (se)
74
+ 1.0000 (0.0000) <= Log. Reg.
75
+ 0.9792 (0.0221) <= alg-1
76
+ 0.9744 (0.0246) <= forest
77
+
78
+ The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
79
+
80
+ >>> perf_error = clone(perf)
81
+ >>> perf_error.error_func = lambda y, hy: (y != hy).mean()
82
+ >>> perf_error
83
+ <Perf>
84
+ Statistic with its standard error (se)
85
+ statistic (se)
86
+ 0.0000 (0.0000) <= Log. Reg.
87
+ 0.0222 (0.0237) <= alg-1
88
+ 0.0222 (0.0215) <= forest
89
+
90
+ """
91
+ def __init__(self, y_true, *y_pred,
92
+ score_func=balanced_accuracy_score,
93
+ error_func=None,
94
+ num_samples: int=500,
95
+ n_jobs: int=-1,
96
+ use_tqdm=True,
97
+ **kwargs):
98
+ assert (score_func is None) ^ (error_func is None)
99
+ self.score_func = score_func
100
+ self.error_func = error_func
101
+ algs = {}
102
+ for k, v in enumerate(y_pred):
103
+ algs[f'alg-{k+1}'] = np.asanyarray(v)
104
+ algs.update(**kwargs)
105
+ self.predictions = algs
106
+ self.y_true = y_true
107
+ self.num_samples = num_samples
108
+ self.n_jobs = n_jobs
109
+ self.use_tqdm = use_tqdm
110
+ self._init()
111
+
112
+ def _init(self):
113
+ """Compute the bootstrap statistic"""
114
+
115
+ bib = True if self.score_func is not None else False
116
+ if hasattr(self, '_statistic_samples'):
117
+ _ = self.statistic_samples
118
+ _.BiB = bib
119
+ else:
120
+ _ = StatisticSamples(statistic=self.statistic_func,
121
+ n_jobs=self.n_jobs,
122
+ num_samples=self.num_samples,
123
+ BiB=bib)
124
+ _.samples(N=self.y_true.shape[0])
125
+ self.statistic_samples = _
126
+
127
+ def get_params(self):
128
+ """Parameters"""
129
+
130
+ return dict(y_true=self.y_true,
131
+ score_func=self.score_func,
132
+ error_func=self.error_func,
133
+ num_samples=self.num_samples,
134
+ n_jobs=self.n_jobs)
135
+
136
+ def __sklearn_clone__(self):
137
+ klass = self.__class__
138
+ params = self.get_params()
139
+ ins = klass(**params)
140
+ ins.predictions = dict(self.predictions)
141
+ ins._statistic_samples._samples = self.statistic_samples._samples
142
+ return ins
143
+
144
+ def __repr__(self):
145
+ """Prediction statistics with standard error in parenthesis"""
146
+ return f"<{self.__class__.__name__}>\n{self}"
147
+
148
+ def __str__(self):
149
+ """Prediction statistics with standard error in parenthesis"""
150
+
151
+ se = self.se
152
+ output = ["Statistic with its standard error (se)"]
153
+ output.append("statistic (se)")
154
+ for key, value in self.statistic.items():
155
+ output.append(f'{value:0.4f} ({se[key]:0.4f}) <= {key}')
156
+ return "\n".join(output)
157
+
158
+ def __call__(self, y_pred, name=None):
159
+ """Add predictions"""
160
+ if name is None:
161
+ k = len(self.predictions) + 1
162
+ if k == 0:
163
+ k = 1
164
+ name = f'alg-{k}'
165
+ self.predictions[name] = np.asanyarray(y_pred)
166
+ samples = self._statistic_samples
167
+ calls = samples.calls
168
+ if name in calls:
169
+ del calls[name]
170
+ return self
171
+
172
+ def difference(self, wrt_to: str=None):
173
+ """Compute the difference w.r.t any algorithm by default is the best
174
+
175
+ >>> from sklearn.svm import LinearSVC
176
+ >>> from sklearn.ensemble import RandomForestClassifier
177
+ >>> from sklearn.datasets import load_iris
178
+ >>> from sklearn.model_selection import train_test_split
179
+ >>> from sklearn.base import clone
180
+ >>> from CompStats.interface import Perf
181
+ >>> X, y = load_iris(return_X_y=True)
182
+ >>> _ = train_test_split(X, y, test_size=0.3)
183
+ >>> X_train, X_val, y_train, y_val = _
184
+ >>> m = LinearSVC().fit(X_train, y_train)
185
+ >>> hy = m.predict(X_val)
186
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
187
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
188
+ >>> perf.difference()
189
+ <Difference>
190
+ difference p-values w.r.t alg-1
191
+ forest 0.06
192
+ """
193
+ if wrt_to is None:
194
+ wrt_to = self.best[0]
195
+ base = self.statistic_samples.calls[wrt_to]
196
+ sign = 1 if self.statistic_samples.BiB else -1
197
+ diff = dict()
198
+ for k, v in self.statistic_samples.calls.items():
199
+ if k == wrt_to:
200
+ continue
201
+ diff[k] = sign * (base - v)
202
+ diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
203
+ statistic=self.statistic,
204
+ best=self.best[0])
205
+ diff_ins.statistic_samples.calls = diff
206
+ diff_ins.statistic_samples.info['best'] = self.best[0]
207
+ return diff_ins
208
+
209
+ @property
210
+ def best(self):
211
+ """Best system"""
212
+
213
+ try:
214
+ return self._best
215
+ except AttributeError:
216
+ statistic = [(k, v) for k, v in self.statistic.items()]
217
+ statistic = sorted(statistic, key=lambda x: x[1],
218
+ reverse=self.statistic_samples.BiB)
219
+ self._best = statistic[0]
220
+ return self._best
221
+
222
+ @property
223
+ def statistic(self):
224
+ """Statistic
225
+
226
+ >>> from sklearn.svm import LinearSVC
227
+ >>> from sklearn.ensemble import RandomForestClassifier
228
+ >>> from sklearn.datasets import load_iris
229
+ >>> from sklearn.model_selection import train_test_split
230
+ >>> from CompStats.interface import Perf
231
+ >>> X, y = load_iris(return_X_y=True)
232
+ >>> _ = train_test_split(X, y, test_size=0.3)
233
+ >>> X_train, X_val, y_train, y_val = _
234
+ >>> m = LinearSVC().fit(X_train, y_train)
235
+ >>> hy = m.predict(X_val)
236
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
237
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
238
+ >>> perf.statistic
239
+ {'alg-1': 1.0, 'forest': 0.9500891265597148}
240
+ """
241
+
242
+ data = sorted([(k, self.statistic_func(self.y_true, v))
243
+ for k, v in self.predictions.items()],
244
+ key=lambda x: x[1], reverse=self.statistic_samples.BiB)
245
+ return dict(data)
246
+
247
+ @property
248
+ def se(self):
249
+ """Standard Error
250
+
251
+ >>> from sklearn.svm import LinearSVC
252
+ >>> from sklearn.ensemble import RandomForestClassifier
253
+ >>> from sklearn.datasets import load_iris
254
+ >>> from sklearn.model_selection import train_test_split
255
+ >>> from CompStats.interface import Perf
256
+ >>> X, y = load_iris(return_X_y=True)
257
+ >>> _ = train_test_split(X, y, test_size=0.3)
258
+ >>> X_train, X_val, y_train, y_val = _
259
+ >>> m = LinearSVC().fit(X_train, y_train)
260
+ >>> hy = m.predict(X_val)
261
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
262
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
263
+ >>> perf.se
264
+ {'alg-1': 0.0, 'forest': 0.026945730782184187}
265
+ """
266
+
267
+ return SE(self.statistic_samples)
268
+
269
+ def plot(self, **kwargs):
270
+ """plot with seaborn
271
+
272
+ >>> from sklearn.svm import LinearSVC
273
+ >>> from sklearn.ensemble import RandomForestClassifier
274
+ >>> from sklearn.datasets import load_iris
275
+ >>> from sklearn.model_selection import train_test_split
276
+ >>> from CompStats.interface import Perf
277
+ >>> X, y = load_iris(return_X_y=True)
278
+ >>> _ = train_test_split(X, y, test_size=0.3)
279
+ >>> X_train, X_val, y_train, y_val = _
280
+ >>> m = LinearSVC().fit(X_train, y_train)
281
+ >>> hy = m.predict(X_val)
282
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
283
+ >>> perf = Perf(y_val, hy, score_func=None,
284
+ error_func=lambda y, hy: (y != hy).mean(),
285
+ forest=ens.predict(X_val))
286
+ >>> perf.plot()
287
+ """
288
+ if self.score_func is not None:
289
+ value_name = 'Score'
290
+ else:
291
+ value_name = 'Error'
292
+ _ = dict(value_name=value_name)
293
+ _.update(kwargs)
294
+ return plot_performance(self.statistic_samples, **_)
295
+
296
+ @property
297
+ def n_jobs(self):
298
+ """Number of jobs to compute the statistics"""
299
+ return self._n_jobs
300
+
301
+ @n_jobs.setter
302
+ def n_jobs(self, value):
303
+ self._n_jobs = value
304
+
305
+ @property
306
+ def statistic_func(self):
307
+ """Statistic function"""
308
+ if self.score_func is not None:
309
+ return self.score_func
310
+ return self.error_func
311
+
312
+ @property
313
+ def statistic_samples(self):
314
+ """Statistic Samples"""
315
+
316
+ samples = self._statistic_samples
317
+ algs = set(samples.calls.keys())
318
+ algs = set(self.predictions.keys()) - algs
319
+ if len(algs):
320
+ for key in progress_bar(algs, use_tqdm=self.use_tqdm):
321
+ samples(self.y_true, self.predictions[key], name=key)
322
+ return self._statistic_samples
323
+
324
+ @statistic_samples.setter
325
+ def statistic_samples(self, value):
326
+ self._statistic_samples = value
327
+
328
+ @property
329
+ def num_samples(self):
330
+ """Number of bootstrap samples"""
331
+ return self._num_samples
332
+
333
+ @num_samples.setter
334
+ def num_samples(self, value):
335
+ self._num_samples = value
336
+
337
+ @property
338
+ def predictions(self):
339
+ """Predictions"""
340
+ return self._predictions
341
+
342
+ @predictions.setter
343
+ def predictions(self, value):
344
+ self._predictions = value
345
+
346
+ @property
347
+ def y_true(self):
348
+ """True output, gold standard o :math:`y`"""
349
+
350
+ return self._y_true
351
+
352
+ @y_true.setter
353
+ def y_true(self, value):
354
+ if isinstance(value, pd.DataFrame):
355
+ self._y_true = value['y'].to_numpy()
356
+ algs = {}
357
+ for c in value.columns:
358
+ if c == 'y':
359
+ continue
360
+ algs[c] = value[c].to_numpy()
361
+ self.predictions.update(algs)
362
+ return
363
+ self._y_true = value
364
+
365
+ @property
366
+ def score_func(self):
367
+ """Score function"""
368
+ return self._score_func
369
+
370
+ @score_func.setter
371
+ def score_func(self, value):
372
+ self._score_func = value
373
+ if value is not None:
374
+ self.error_func = None
375
+ if hasattr(self, '_statistic_samples'):
376
+ self._statistic_samples.statistic = value
377
+ self._statistic_samples.BiB = True
378
+
379
+ @property
380
+ def error_func(self):
381
+ """Error function"""
382
+ return self._error_func
383
+
384
+ @error_func.setter
385
+ def error_func(self, value):
386
+ self._error_func = value
387
+ if value is not None:
388
+ self.score_func = None
389
+ if hasattr(self, '_statistic_samples'):
390
+ self._statistic_samples.statistic = value
391
+ self._statistic_samples.BiB = False
392
+
393
+
394
+ @dataclass
395
+ class Difference:
396
+ """Difference
397
+
398
+ >>> from sklearn.svm import LinearSVC
399
+ >>> from sklearn.ensemble import RandomForestClassifier
400
+ >>> from sklearn.datasets import load_iris
401
+ >>> from sklearn.model_selection import train_test_split
402
+ >>> from sklearn.base import clone
403
+ >>> from CompStats.interface import Perf
404
+ >>> X, y = load_iris(return_X_y=True)
405
+ >>> _ = train_test_split(X, y, test_size=0.3)
406
+ >>> X_train, X_val, y_train, y_val = _
407
+ >>> m = LinearSVC().fit(X_train, y_train)
408
+ >>> hy = m.predict(X_val)
409
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
410
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
411
+ >>> diff = perf.difference()
412
+ >>> diff
413
+ <Difference>
414
+ difference p-values w.r.t alg-1
415
+ 0.0780 <= forest
416
+ """
417
+
418
+ statistic_samples:StatisticSamples=None
419
+ best:str=None
420
+ statistic:dict=None
421
+
422
+ def __repr__(self):
423
+ """p-value"""
424
+ return f"<{self.__class__.__name__}>\n{self}"
425
+
426
+ def __str__(self):
427
+ """p-value"""
428
+ output = [f"difference p-values w.r.t {self.best}"]
429
+ for k, v in self.p_value().items():
430
+ output.append(f'{v:0.4f} <= {k}')
431
+ return "\n".join(output)
432
+
433
+ def p_value(self):
434
+ """Compute p_value of the differences
435
+
436
+ >>> from sklearn.svm import LinearSVC
437
+ >>> from sklearn.ensemble import RandomForestClassifier
438
+ >>> from sklearn.datasets import load_iris
439
+ >>> from sklearn.model_selection import train_test_split
440
+ >>> from sklearn.base import clone
441
+ >>> from CompStats.interface import Perf
442
+ >>> X, y = load_iris(return_X_y=True)
443
+ >>> _ = train_test_split(X, y, test_size=0.3)
444
+ >>> X_train, X_val, y_train, y_val = _
445
+ >>> m = LinearSVC().fit(X_train, y_train)
446
+ >>> hy = m.predict(X_val)
447
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
448
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
449
+ >>> diff = perf.difference()
450
+ >>> diff.p_value()
451
+ {'forest': np.float64(0.3)}
452
+ """
453
+ values = []
454
+ sign = 1 if self.statistic_samples.BiB else -1
455
+ for k, v in self.statistic_samples.calls.items():
456
+ delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
457
+ values.append((k, (v > delta).mean()))
458
+ values.sort(key=lambda x: x[1])
459
+ return dict(values)
460
+
461
+ def plot(self, **kwargs):
462
+ """Plot
463
+
464
+ >>> from sklearn.svm import LinearSVC
465
+ >>> from sklearn.ensemble import RandomForestClassifier
466
+ >>> from sklearn.datasets import load_iris
467
+ >>> from sklearn.model_selection import train_test_split
468
+ >>> from sklearn.base import clone
469
+ >>> from CompStats.interface import Perf
470
+ >>> X, y = load_iris(return_X_y=True)
471
+ >>> _ = train_test_split(X, y, test_size=0.3)
472
+ >>> X_train, X_val, y_train, y_val = _
473
+ >>> m = LinearSVC().fit(X_train, y_train)
474
+ >>> hy = m.predict(X_val)
475
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
476
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
477
+ >>> diff = perf.difference()
478
+ >>> diff.plot()
479
+ """
480
+
481
+ return plot_difference(self.statistic_samples, **kwargs)