CompStats 0.1.5__tar.gz → 0.1.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/__init__.py +1 -1
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/bootstrap.py +4 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/interface.py +164 -34
- compstats-0.1.7/CompStats/tests/test_interface.py +310 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_metrics.py +29 -76
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/utils.py +26 -1
- {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/PKG-INFO +18 -18
- {compstats-0.1.5 → compstats-0.1.7}/PKG-INFO +18 -18
- {compstats-0.1.5 → compstats-0.1.7}/README.rst +17 -17
- compstats-0.1.5/CompStats/tests/test_interface.py +0 -153
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/measurements.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/metrics.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/performance.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/LICENSE +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/MANIFEST.in +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/pyproject.toml +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/setup.cfg +0 -0
- {compstats-0.1.5 → compstats-0.1.7}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.7'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -18,8 +18,10 @@ import pandas as pd
|
|
|
18
18
|
import numpy as np
|
|
19
19
|
from CompStats.bootstrap import StatisticSamples
|
|
20
20
|
from CompStats.utils import progress_bar
|
|
21
|
+
from CompStats import measurements
|
|
21
22
|
from CompStats.measurements import SE
|
|
22
23
|
from CompStats.performance import plot_performance, plot_difference
|
|
24
|
+
from CompStats.utils import dataframe
|
|
23
25
|
|
|
24
26
|
|
|
25
27
|
class Perf(object):
|
|
@@ -147,10 +149,21 @@ class Perf(object):
|
|
|
147
149
|
"""Prediction statistics with standard error in parenthesis"""
|
|
148
150
|
arg = 'score_func' if self.error_func is None else 'error_func'
|
|
149
151
|
func_name = self.statistic_func.__name__
|
|
150
|
-
|
|
151
|
-
|
|
152
|
+
statistic = self.statistic
|
|
153
|
+
if isinstance(statistic, dict):
|
|
154
|
+
return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
|
|
155
|
+
elif isinstance(statistic, float):
|
|
156
|
+
return f"<{self.__class__.__name__}({arg}={func_name}, statistic={statistic:0.4f}, se={self.se:0.4f})>"
|
|
157
|
+
desc = [f'{k:0.4f}' for k in statistic]
|
|
158
|
+
desc = ', '.join(desc)
|
|
159
|
+
desc_se = [f'{k:0.4f}' for k in self.se]
|
|
160
|
+
desc_se = ', '.join(desc_se)
|
|
161
|
+
return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
|
|
162
|
+
|
|
152
163
|
def __str__(self):
|
|
153
164
|
"""Prediction statistics with standard error in parenthesis"""
|
|
165
|
+
if not isinstance(self.statistic, dict):
|
|
166
|
+
return self.__repr__()
|
|
154
167
|
|
|
155
168
|
se = self.se
|
|
156
169
|
output = ["Statistic with its standard error (se)"]
|
|
@@ -173,6 +186,7 @@ class Perf(object):
|
|
|
173
186
|
if k == 0:
|
|
174
187
|
k = 1
|
|
175
188
|
name = f'alg-{k}'
|
|
189
|
+
self.best = None
|
|
176
190
|
self.predictions[name] = np.asanyarray(y_pred)
|
|
177
191
|
samples = self._statistic_samples
|
|
178
192
|
calls = samples.calls
|
|
@@ -180,7 +194,7 @@ class Perf(object):
|
|
|
180
194
|
del calls[name]
|
|
181
195
|
return self
|
|
182
196
|
|
|
183
|
-
def difference(self,
|
|
197
|
+
def difference(self, wrt: str=None):
|
|
184
198
|
"""Compute the difference w.r.t any algorithm by default is the best
|
|
185
199
|
|
|
186
200
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -201,37 +215,60 @@ class Perf(object):
|
|
|
201
215
|
difference p-values w.r.t alg-1
|
|
202
216
|
forest 0.06
|
|
203
217
|
"""
|
|
204
|
-
if
|
|
205
|
-
|
|
206
|
-
|
|
218
|
+
if wrt is None:
|
|
219
|
+
wrt = self.best
|
|
220
|
+
if isinstance(wrt, str):
|
|
221
|
+
base = self.statistic_samples.calls[wrt]
|
|
222
|
+
else:
|
|
223
|
+
base = np.array([self.statistic_samples.calls[key][:, col]
|
|
224
|
+
for col, key in enumerate(wrt)]).T
|
|
207
225
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
208
226
|
diff = dict()
|
|
209
227
|
for k, v in self.statistic_samples.calls.items():
|
|
210
|
-
if k ==
|
|
228
|
+
if base.ndim == 1 and k == wrt:
|
|
211
229
|
continue
|
|
212
230
|
diff[k] = sign * (base - v)
|
|
213
231
|
diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
|
|
214
|
-
statistic=self.statistic
|
|
215
|
-
best=self.best[0])
|
|
232
|
+
statistic=self.statistic)
|
|
216
233
|
diff_ins.sorting_func = self.sorting_func
|
|
217
234
|
diff_ins.statistic_samples.calls = diff
|
|
218
|
-
diff_ins.statistic_samples.info['best'] = self.best
|
|
235
|
+
diff_ins.statistic_samples.info['best'] = self.best
|
|
236
|
+
diff_ins.best = self.best
|
|
219
237
|
return diff_ins
|
|
220
238
|
|
|
221
239
|
@property
|
|
222
240
|
def best(self):
|
|
223
|
-
"""
|
|
224
|
-
|
|
225
|
-
try:
|
|
241
|
+
"""System with best performance"""
|
|
242
|
+
if hasattr(self, '_best') and self._best is not None:
|
|
226
243
|
return self._best
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
244
|
+
if not isinstance(self.statistic, dict):
|
|
245
|
+
key, value = list(self.statistic_samples.calls.items())[0]
|
|
246
|
+
if value.ndim == 1:
|
|
247
|
+
self._best = key
|
|
248
|
+
else:
|
|
249
|
+
self._best = np.array([key] * value.shape[1])
|
|
250
|
+
return self._best
|
|
251
|
+
BiB = True if self.statistic_samples.BiB else False
|
|
252
|
+
keys = np.array(list(self.statistic.keys()))
|
|
253
|
+
data = np.asanyarray([self.statistic[k]
|
|
254
|
+
for k in keys])
|
|
255
|
+
if isinstance(self.statistic[keys[0]], np.ndarray):
|
|
256
|
+
if BiB:
|
|
257
|
+
best = data.argmax(axis=0)
|
|
258
|
+
else:
|
|
259
|
+
best = data.argmin(axis=0)
|
|
260
|
+
else:
|
|
261
|
+
if BiB:
|
|
262
|
+
best = data.argmax()
|
|
263
|
+
else:
|
|
264
|
+
best = data.argmin()
|
|
265
|
+
self._best = keys[best]
|
|
233
266
|
return self._best
|
|
234
267
|
|
|
268
|
+
@best.setter
|
|
269
|
+
def best(self, value):
|
|
270
|
+
self._best = value
|
|
271
|
+
|
|
235
272
|
@property
|
|
236
273
|
def sorting_func(self):
|
|
237
274
|
"""Rank systems when multiple performances are used"""
|
|
@@ -265,6 +302,8 @@ class Perf(object):
|
|
|
265
302
|
for k, v in self.predictions.items()],
|
|
266
303
|
key=lambda x: self.sorting_func(x[1]),
|
|
267
304
|
reverse=self.statistic_samples.BiB)
|
|
305
|
+
if len(data) == 1:
|
|
306
|
+
return data[0][1]
|
|
268
307
|
return dict(data)
|
|
269
308
|
|
|
270
309
|
@property
|
|
@@ -287,9 +326,19 @@ class Perf(object):
|
|
|
287
326
|
{'alg-1': 0.0, 'forest': 0.026945730782184187}
|
|
288
327
|
"""
|
|
289
328
|
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
329
|
+
output = SE(self.statistic_samples)
|
|
330
|
+
if len(output) == 1:
|
|
331
|
+
return list(output.values())[0]
|
|
332
|
+
return output
|
|
333
|
+
|
|
334
|
+
def plot(self, value_name:str=None,
|
|
335
|
+
var_name:str='Performance',
|
|
336
|
+
alg_legend:str='Algorithm',
|
|
337
|
+
perf_names:list=None,
|
|
338
|
+
CI:float=0.05,
|
|
339
|
+
kind:str='point', linestyle:str='none',
|
|
340
|
+
col_wrap:int=3, capsize:float=0.2,
|
|
341
|
+
**kwargs):
|
|
293
342
|
"""plot with seaborn
|
|
294
343
|
|
|
295
344
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -308,13 +357,37 @@ class Perf(object):
|
|
|
308
357
|
forest=ens.predict(X_val))
|
|
309
358
|
>>> perf.plot()
|
|
310
359
|
"""
|
|
360
|
+
import seaborn as sns
|
|
311
361
|
if self.score_func is not None:
|
|
312
362
|
value_name = 'Score'
|
|
313
363
|
else:
|
|
314
364
|
value_name = 'Error'
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
365
|
+
df = self.dataframe(value_name=value_name, var_name=var_name,
|
|
366
|
+
alg_legend=alg_legend, perf_names=perf_names)
|
|
367
|
+
if var_name not in df.columns:
|
|
368
|
+
var_name = None
|
|
369
|
+
col_wrap = None
|
|
370
|
+
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
371
|
+
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
372
|
+
y=alg_legend, col=var_name,
|
|
373
|
+
kind=kind, linestyle=linestyle,
|
|
374
|
+
col_wrap=col_wrap, capsize=capsize, **kwargs)
|
|
375
|
+
return f_grid
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def dataframe(self, value_name:str='Score',
|
|
379
|
+
var_name:str='Performance',
|
|
380
|
+
alg_legend:str='Algorithm',
|
|
381
|
+
perf_names:str=None):
|
|
382
|
+
"""Dataframe"""
|
|
383
|
+
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
384
|
+
func_name = self.statistic_func.__name__
|
|
385
|
+
perf_names = [f'{func_name}({i})'
|
|
386
|
+
for i, k in enumerate(self.best)]
|
|
387
|
+
return dataframe(self, value_name=value_name,
|
|
388
|
+
var_name=var_name,
|
|
389
|
+
alg_legend=alg_legend,
|
|
390
|
+
perf_names=perf_names)
|
|
318
391
|
|
|
319
392
|
@property
|
|
320
393
|
def n_jobs(self):
|
|
@@ -439,8 +512,8 @@ class Difference:
|
|
|
439
512
|
"""
|
|
440
513
|
|
|
441
514
|
statistic_samples:StatisticSamples=None
|
|
442
|
-
best:str=None
|
|
443
515
|
statistic:dict=None
|
|
516
|
+
best:str=None
|
|
444
517
|
|
|
445
518
|
@property
|
|
446
519
|
def sorting_func(self):
|
|
@@ -453,11 +526,19 @@ class Difference:
|
|
|
453
526
|
|
|
454
527
|
def __repr__(self):
|
|
455
528
|
"""p-value"""
|
|
456
|
-
return f"<{self.__class__.__name__}>\n{self}"
|
|
529
|
+
return f"<{self.__class__.__name__}>\n{self}"
|
|
457
530
|
|
|
458
531
|
def __str__(self):
|
|
459
532
|
"""p-value"""
|
|
460
|
-
|
|
533
|
+
if isinstance(self.best, str):
|
|
534
|
+
best = f' w.r.t {self.best}'
|
|
535
|
+
else:
|
|
536
|
+
best = ''
|
|
537
|
+
output = [f"difference p-values {best}"]
|
|
538
|
+
best = self.best
|
|
539
|
+
if isinstance(best, np.ndarray):
|
|
540
|
+
desc = ', '.join(best)
|
|
541
|
+
output.append(f'{desc} <= Best')
|
|
461
542
|
for key, value in self.p_value().items():
|
|
462
543
|
if isinstance(value, float):
|
|
463
544
|
output.append(f'{value:0.4f} <= {key}')
|
|
@@ -465,8 +546,20 @@ class Difference:
|
|
|
465
546
|
desc = [f'{v:0.4f}' for v in value]
|
|
466
547
|
desc = ', '.join(desc)
|
|
467
548
|
desc = f'{desc} <= {key}'
|
|
549
|
+
output.append(desc)
|
|
468
550
|
return "\n".join(output)
|
|
469
551
|
|
|
552
|
+
def _delta_best(self):
|
|
553
|
+
"""Compute multiple delta"""
|
|
554
|
+
if isinstance(self.best, str):
|
|
555
|
+
return self.statistic[self.best]
|
|
556
|
+
keys = np.unique(self.best)
|
|
557
|
+
statistic = np.array([self.statistic[k]
|
|
558
|
+
for k in keys])
|
|
559
|
+
m = {v: k for k, v in enumerate(keys)}
|
|
560
|
+
best = np.array([m[x] for x in self.best])
|
|
561
|
+
return statistic[best, np.arange(best.shape[0])]
|
|
562
|
+
|
|
470
563
|
def p_value(self, right:bool=True):
|
|
471
564
|
"""Compute p_value of the differences
|
|
472
565
|
|
|
@@ -492,10 +585,10 @@ class Difference:
|
|
|
492
585
|
"""
|
|
493
586
|
values = []
|
|
494
587
|
sign = 1 if self.statistic_samples.BiB else -1
|
|
495
|
-
|
|
588
|
+
delta_best = self._delta_best()
|
|
496
589
|
for k, v in self.statistic_samples.calls.items():
|
|
497
|
-
delta = 2 * sign * (
|
|
498
|
-
if
|
|
590
|
+
delta = 2 * sign * (delta_best - self.statistic[k])
|
|
591
|
+
if not isinstance(delta_best, np.ndarray):
|
|
499
592
|
if right:
|
|
500
593
|
values.append((k, (v >= delta).mean()))
|
|
501
594
|
else:
|
|
@@ -508,7 +601,28 @@ class Difference:
|
|
|
508
601
|
values.sort(key=lambda x: self.sorting_func(x[1]))
|
|
509
602
|
return dict(values)
|
|
510
603
|
|
|
511
|
-
def
|
|
604
|
+
def dataframe(self, value_name:str='Score',
|
|
605
|
+
var_name:str='Best',
|
|
606
|
+
alg_legend:str='Algorithm',
|
|
607
|
+
perf_names:str=None):
|
|
608
|
+
"""Dataframe"""
|
|
609
|
+
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
610
|
+
perf_names = [f'{alg}({k})'
|
|
611
|
+
for k, alg in enumerate(self.best)]
|
|
612
|
+
return dataframe(self, value_name=value_name,
|
|
613
|
+
var_name=var_name,
|
|
614
|
+
alg_legend=alg_legend,
|
|
615
|
+
perf_names=perf_names)
|
|
616
|
+
|
|
617
|
+
def plot(self, value_name:str='Difference',
|
|
618
|
+
var_name:str='Best',
|
|
619
|
+
alg_legend:str='Algorithm',
|
|
620
|
+
perf_names:list=None,
|
|
621
|
+
CI:float=0.05,
|
|
622
|
+
kind:str='point', linestyle:str='none',
|
|
623
|
+
col_wrap:int=3, capsize:float=0.2,
|
|
624
|
+
set_refline:bool=True,
|
|
625
|
+
**kwargs):
|
|
512
626
|
"""Plot
|
|
513
627
|
|
|
514
628
|
>>> from sklearn.svm import LinearSVC
|
|
@@ -527,5 +641,21 @@ class Difference:
|
|
|
527
641
|
>>> diff = perf.difference()
|
|
528
642
|
>>> diff.plot()
|
|
529
643
|
"""
|
|
530
|
-
|
|
531
|
-
|
|
644
|
+
import seaborn as sns
|
|
645
|
+
df = self.dataframe(value_name=value_name,
|
|
646
|
+
var_name=var_name,
|
|
647
|
+
alg_legend=alg_legend, perf_names=perf_names)
|
|
648
|
+
title = var_name
|
|
649
|
+
if var_name not in df.columns:
|
|
650
|
+
var_name = None
|
|
651
|
+
col_wrap = None
|
|
652
|
+
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
653
|
+
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
654
|
+
y=alg_legend, col=var_name,
|
|
655
|
+
kind=kind, linestyle=linestyle,
|
|
656
|
+
col_wrap=col_wrap, capsize=capsize, **kwargs)
|
|
657
|
+
if set_refline:
|
|
658
|
+
f_grid.refline(x=0)
|
|
659
|
+
if isinstance(self.best, str):
|
|
660
|
+
f_grid.facet_axis(0, 0).set_title(f'{title} = {self.best}')
|
|
661
|
+
return f_grid
|
|
@@ -0,0 +1,310 @@
|
|
|
1
|
+
# Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
|
|
2
|
+
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from sklearn.base import clone
|
|
17
|
+
from sklearn.svm import LinearSVC
|
|
18
|
+
from sklearn.ensemble import RandomForestClassifier
|
|
19
|
+
from sklearn.naive_bayes import GaussianNB
|
|
20
|
+
from sklearn.datasets import load_iris, load_digits
|
|
21
|
+
from sklearn.model_selection import train_test_split
|
|
22
|
+
import pandas as pd
|
|
23
|
+
from CompStats.tests.test_performance import DATA
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_Difference_dataframe():
|
|
27
|
+
"""Test Difference dataframe"""
|
|
28
|
+
from CompStats.metrics import f1_score
|
|
29
|
+
|
|
30
|
+
X, y = load_digits(return_X_y=True)
|
|
31
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
32
|
+
X_train, X_val, y_train, y_val = _
|
|
33
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
34
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
35
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
36
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
37
|
+
average=None,
|
|
38
|
+
num_samples=50)
|
|
39
|
+
score(nb.predict(X_val))
|
|
40
|
+
score(svm.predict(X_val))
|
|
41
|
+
diff = score.difference()
|
|
42
|
+
df = diff.dataframe()
|
|
43
|
+
assert 'Best' in df.columns
|
|
44
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
45
|
+
average='macro',
|
|
46
|
+
num_samples=50)
|
|
47
|
+
score(nb.predict(X_val))
|
|
48
|
+
score(svm.predict(X_val))
|
|
49
|
+
diff = score.difference()
|
|
50
|
+
df = diff.dataframe()
|
|
51
|
+
assert 'Best' not in df.columns
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_Perf_dataframe():
|
|
55
|
+
"""Test Perf dataframe"""
|
|
56
|
+
from CompStats.metrics import f1_score
|
|
57
|
+
|
|
58
|
+
X, y = load_digits(return_X_y=True)
|
|
59
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
60
|
+
X_train, X_val, y_train, y_val = _
|
|
61
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
62
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
63
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
64
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
65
|
+
average=None,
|
|
66
|
+
num_samples=50)
|
|
67
|
+
df = score.dataframe()
|
|
68
|
+
score(nb.predict(X_val))
|
|
69
|
+
score(svm.predict(X_val))
|
|
70
|
+
df = score.dataframe()
|
|
71
|
+
assert 'Performance' in df.columns
|
|
72
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
73
|
+
average='macro',
|
|
74
|
+
num_samples=50)
|
|
75
|
+
score(nb.predict(X_val))
|
|
76
|
+
score(svm.predict(X_val))
|
|
77
|
+
df = score.dataframe()
|
|
78
|
+
assert 'Performance' not in df.columns
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_Perf_plot_multi():
|
|
82
|
+
"""Test Perf plot multiple"""
|
|
83
|
+
from CompStats.metrics import f1_score
|
|
84
|
+
|
|
85
|
+
X, y = load_digits(return_X_y=True)
|
|
86
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
87
|
+
X_train, X_val, y_train, y_val = _
|
|
88
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
89
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
90
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
91
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
92
|
+
average=None,
|
|
93
|
+
num_samples=50)
|
|
94
|
+
score(nb.predict(X_val))
|
|
95
|
+
score(svm.predict(X_val))
|
|
96
|
+
f_grid = score.plot()
|
|
97
|
+
assert f_grid is not None
|
|
98
|
+
|
|
99
|
+
def test_Perf_statistic_one():
|
|
100
|
+
"""Test Perf statistic one alg"""
|
|
101
|
+
from CompStats.metrics import f1_score
|
|
102
|
+
|
|
103
|
+
X, y = load_digits(return_X_y=True)
|
|
104
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
105
|
+
X_train, X_val, y_train, y_val = _
|
|
106
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
107
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
108
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
109
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
110
|
+
average=None,
|
|
111
|
+
num_samples=50)
|
|
112
|
+
assert isinstance(score.statistic, np.ndarray)
|
|
113
|
+
assert isinstance(str(score), str)
|
|
114
|
+
score = f1_score(y_val, ens.predict(X_val),
|
|
115
|
+
average='macro',
|
|
116
|
+
num_samples=50)
|
|
117
|
+
assert isinstance(score.statistic, float)
|
|
118
|
+
assert isinstance(str(score), str)
|
|
119
|
+
assert isinstance(score.se, float)
|
|
120
|
+
|
|
121
|
+
def test_Perf_best():
|
|
122
|
+
"""Test Perf best"""
|
|
123
|
+
from CompStats.metrics import f1_score
|
|
124
|
+
|
|
125
|
+
X, y = load_digits(return_X_y=True)
|
|
126
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
127
|
+
X_train, X_val, y_train, y_val = _
|
|
128
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
129
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
130
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
131
|
+
score = f1_score(y_val, average=None,
|
|
132
|
+
num_samples=50)
|
|
133
|
+
score(ens.predict(X_val), name='forest')
|
|
134
|
+
score(nb.predict(X_val), name='NB')
|
|
135
|
+
score(svm.predict(X_val), name='svm')
|
|
136
|
+
assert isinstance(score.best, np.ndarray)
|
|
137
|
+
score = f1_score(y_val, average='macro',
|
|
138
|
+
num_samples=50)
|
|
139
|
+
score(ens.predict(X_val), name='forest')
|
|
140
|
+
score(nb.predict(X_val), name='NB')
|
|
141
|
+
score(svm.predict(X_val), name='svm')
|
|
142
|
+
assert isinstance(score.best, str)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def test_difference_best():
|
|
146
|
+
"""Test multiple performance measures"""
|
|
147
|
+
from CompStats.metrics import f1_score
|
|
148
|
+
|
|
149
|
+
X, y = load_digits(return_X_y=True)
|
|
150
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
151
|
+
X_train, X_val, y_train, y_val = _
|
|
152
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
153
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
154
|
+
svm = LinearSVC().fit(X_train, y_train)
|
|
155
|
+
score = f1_score(y_val, average=None,
|
|
156
|
+
num_samples=50)
|
|
157
|
+
score(ens.predict(X_val), name='forest')
|
|
158
|
+
score(nb.predict(X_val), name='NB')
|
|
159
|
+
score(svm.predict(X_val), name='svm')
|
|
160
|
+
diff = score.difference()
|
|
161
|
+
assert isinstance(diff.best, np.ndarray)
|
|
162
|
+
score = f1_score(y_val, average='macro',
|
|
163
|
+
num_samples=50)
|
|
164
|
+
score(ens.predict(X_val), name='forest')
|
|
165
|
+
score(nb.predict(X_val), name='NB')
|
|
166
|
+
score(svm.predict(X_val), name='svm')
|
|
167
|
+
diff = score.difference()
|
|
168
|
+
assert isinstance(diff.best, str)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_difference_str__():
|
|
172
|
+
"""Test f1_score"""
|
|
173
|
+
from CompStats.metrics import f1_score
|
|
174
|
+
|
|
175
|
+
X, y = load_iris(return_X_y=True)
|
|
176
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
177
|
+
X_train, X_val, y_train, y_val = _
|
|
178
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
179
|
+
nb = GaussianNB().fit(X_train, y_train)
|
|
180
|
+
perf = f1_score(y_val, nb.predict(X_val),
|
|
181
|
+
forest=ens.predict(X_val),
|
|
182
|
+
num_samples=50, average=None)
|
|
183
|
+
diff = perf.difference()
|
|
184
|
+
p_values = diff.p_value(right=False)
|
|
185
|
+
dd = list(p_values.values())[0]
|
|
186
|
+
assert isinstance(dd, np.ndarray)
|
|
187
|
+
for average in ['macro', None]:
|
|
188
|
+
perf = f1_score(y_val, nb.predict(X_val),
|
|
189
|
+
forest=ens.predict(X_val),
|
|
190
|
+
num_samples=50, average=average)
|
|
191
|
+
diff = perf.difference()
|
|
192
|
+
print(diff)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def test_Perf():
|
|
196
|
+
"""Test perf"""
|
|
197
|
+
from CompStats.interface import Perf
|
|
198
|
+
|
|
199
|
+
X, y = load_iris(return_X_y=True)
|
|
200
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
201
|
+
X_train, X_val, y_train, y_val = _
|
|
202
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
203
|
+
hy = m.predict(X_val)
|
|
204
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
205
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
206
|
+
assert 'alg-1' in perf.predictions
|
|
207
|
+
assert 'forest' in perf.predictions
|
|
208
|
+
assert str(perf) is not None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def test_Perf_statistic():
|
|
212
|
+
"""Test statistic"""
|
|
213
|
+
from CompStats.interface import Perf
|
|
214
|
+
|
|
215
|
+
X, y = load_iris(return_X_y=True)
|
|
216
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
217
|
+
X_train, X_val, y_train, y_val = _
|
|
218
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
219
|
+
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
220
|
+
perf(ens.predict(X_val))
|
|
221
|
+
assert 'forest' in perf.statistic
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def test_Perf_plot():
|
|
225
|
+
"""Test plot"""
|
|
226
|
+
|
|
227
|
+
from CompStats.interface import Perf
|
|
228
|
+
|
|
229
|
+
X, y = load_iris(return_X_y=True)
|
|
230
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
231
|
+
X_train, X_val, y_train, y_val = _
|
|
232
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
233
|
+
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
234
|
+
perf.plot()
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def test_Perf_clone():
|
|
238
|
+
"""Test Perf.clone"""
|
|
239
|
+
from CompStats.interface import Perf
|
|
240
|
+
|
|
241
|
+
X, y = load_iris(return_X_y=True)
|
|
242
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
243
|
+
X_train, X_val, y_train, y_val = _
|
|
244
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
245
|
+
perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
|
|
246
|
+
samples = perf.statistic_samples._samples
|
|
247
|
+
perf2 = clone(perf)
|
|
248
|
+
perf2.error_func = lambda y, hy: (y != hy).mean()
|
|
249
|
+
assert 'forest' in perf2.statistic_samples.calls
|
|
250
|
+
assert np.all(samples == perf2.statistic_samples._samples)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def test_Perf_difference():
|
|
254
|
+
"""Test difference"""
|
|
255
|
+
from CompStats.interface import Perf, Difference
|
|
256
|
+
|
|
257
|
+
X, y = load_iris(return_X_y=True)
|
|
258
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
259
|
+
X_train, X_val, y_train, y_val = _
|
|
260
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
261
|
+
hy = m.predict(X_val)
|
|
262
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
263
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
264
|
+
diff = perf.difference()
|
|
265
|
+
assert isinstance(diff, Difference)
|
|
266
|
+
assert isinstance(str(diff), str)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def test_Difference_plot():
|
|
270
|
+
"""Test difference plot"""
|
|
271
|
+
from CompStats.interface import Perf
|
|
272
|
+
|
|
273
|
+
X, y = load_iris(return_X_y=True)
|
|
274
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
275
|
+
X_train, X_val, y_train, y_val = _
|
|
276
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
277
|
+
hy = m.predict(X_val)
|
|
278
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
279
|
+
perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
|
|
280
|
+
diff = perf.difference()
|
|
281
|
+
diff.plot()
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def test_Perf_input_dataframe():
|
|
285
|
+
"""Test Perf with dataframe"""
|
|
286
|
+
from CompStats.interface import Perf
|
|
287
|
+
|
|
288
|
+
df = pd.read_csv(DATA)
|
|
289
|
+
perf = Perf(df, num_samples=50)
|
|
290
|
+
assert 'INGEOTEC' in perf.statistic
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def test_Perf_call():
|
|
294
|
+
"""Test Perf call"""
|
|
295
|
+
from CompStats.interface import Perf
|
|
296
|
+
|
|
297
|
+
X, y = load_iris(return_X_y=True)
|
|
298
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
299
|
+
X_train, X_val, y_train, y_val = _
|
|
300
|
+
m = LinearSVC().fit(X_train, y_train)
|
|
301
|
+
hy = m.predict(X_val)
|
|
302
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
303
|
+
hy2 = ens.predict(X_val)
|
|
304
|
+
perf = Perf(y_val, num_samples=50)
|
|
305
|
+
for xx in [hy, hy2]:
|
|
306
|
+
_ = perf(xx)
|
|
307
|
+
print(_)
|
|
308
|
+
perf(hy, name='alg-2')
|
|
309
|
+
assert 'alg-2' not in perf._statistic_samples.calls
|
|
310
|
+
assert 'alg-1' in perf._statistic_samples.calls
|