CompStats 0.1.5__tar.gz → 0.1.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {compstats-0.1.5 → compstats-0.1.7}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.5 → compstats-0.1.7}/CompStats/bootstrap.py +4 -0
  3. {compstats-0.1.5 → compstats-0.1.7}/CompStats/interface.py +164 -34
  4. compstats-0.1.7/CompStats/tests/test_interface.py +310 -0
  5. {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_metrics.py +29 -76
  6. {compstats-0.1.5 → compstats-0.1.7}/CompStats/utils.py +26 -1
  7. {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/PKG-INFO +18 -18
  8. {compstats-0.1.5 → compstats-0.1.7}/PKG-INFO +18 -18
  9. {compstats-0.1.5 → compstats-0.1.7}/README.rst +17 -17
  10. compstats-0.1.5/CompStats/tests/test_interface.py +0 -153
  11. {compstats-0.1.5 → compstats-0.1.7}/CompStats/measurements.py +0 -0
  12. {compstats-0.1.5 → compstats-0.1.7}/CompStats/metrics.py +0 -0
  13. {compstats-0.1.5 → compstats-0.1.7}/CompStats/performance.py +0 -0
  14. {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/__init__.py +0 -0
  15. {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/data.csv +0 -0
  16. {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_bootstrap.py +0 -0
  17. {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_measurements.py +0 -0
  18. {compstats-0.1.5 → compstats-0.1.7}/CompStats/tests/test_performance.py +0 -0
  19. {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/SOURCES.txt +0 -0
  20. {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/dependency_links.txt +0 -0
  21. {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/requires.txt +0 -0
  22. {compstats-0.1.5 → compstats-0.1.7}/CompStats.egg-info/top_level.txt +0 -0
  23. {compstats-0.1.5 → compstats-0.1.7}/LICENSE +0 -0
  24. {compstats-0.1.5 → compstats-0.1.7}/MANIFEST.in +0 -0
  25. {compstats-0.1.5 → compstats-0.1.7}/pyproject.toml +0 -0
  26. {compstats-0.1.5 → compstats-0.1.7}/setup.cfg +0 -0
  27. {compstats-0.1.5 → compstats-0.1.7}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.5'
14
+ __version__ = '0.1.7'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -142,6 +142,10 @@ class StatisticSamples:
142
142
  return inner(N)
143
143
  except AttributeError:
144
144
  return inner(N)
145
+
146
+ def keys(self):
147
+ """calls keys"""
148
+ return self.calls.keys()
145
149
 
146
150
  def __getitem__(self, key):
147
151
  return self.calls[key]
@@ -18,8 +18,10 @@ import pandas as pd
18
18
  import numpy as np
19
19
  from CompStats.bootstrap import StatisticSamples
20
20
  from CompStats.utils import progress_bar
21
+ from CompStats import measurements
21
22
  from CompStats.measurements import SE
22
23
  from CompStats.performance import plot_performance, plot_difference
24
+ from CompStats.utils import dataframe
23
25
 
24
26
 
25
27
  class Perf(object):
@@ -147,10 +149,21 @@ class Perf(object):
147
149
  """Prediction statistics with standard error in parenthesis"""
148
150
  arg = 'score_func' if self.error_func is None else 'error_func'
149
151
  func_name = self.statistic_func.__name__
150
- return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
151
-
152
+ statistic = self.statistic
153
+ if isinstance(statistic, dict):
154
+ return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
155
+ elif isinstance(statistic, float):
156
+ return f"<{self.__class__.__name__}({arg}={func_name}, statistic={statistic:0.4f}, se={self.se:0.4f})>"
157
+ desc = [f'{k:0.4f}' for k in statistic]
158
+ desc = ', '.join(desc)
159
+ desc_se = [f'{k:0.4f}' for k in self.se]
160
+ desc_se = ', '.join(desc_se)
161
+ return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
162
+
152
163
  def __str__(self):
153
164
  """Prediction statistics with standard error in parenthesis"""
165
+ if not isinstance(self.statistic, dict):
166
+ return self.__repr__()
154
167
 
155
168
  se = self.se
156
169
  output = ["Statistic with its standard error (se)"]
@@ -173,6 +186,7 @@ class Perf(object):
173
186
  if k == 0:
174
187
  k = 1
175
188
  name = f'alg-{k}'
189
+ self.best = None
176
190
  self.predictions[name] = np.asanyarray(y_pred)
177
191
  samples = self._statistic_samples
178
192
  calls = samples.calls
@@ -180,7 +194,7 @@ class Perf(object):
180
194
  del calls[name]
181
195
  return self
182
196
 
183
- def difference(self, wrt_to: str=None):
197
+ def difference(self, wrt: str=None):
184
198
  """Compute the difference w.r.t any algorithm by default is the best
185
199
 
186
200
  >>> from sklearn.svm import LinearSVC
@@ -201,37 +215,60 @@ class Perf(object):
201
215
  difference p-values w.r.t alg-1
202
216
  forest 0.06
203
217
  """
204
- if wrt_to is None:
205
- wrt_to = self.best[0]
206
- base = self.statistic_samples.calls[wrt_to]
218
+ if wrt is None:
219
+ wrt = self.best
220
+ if isinstance(wrt, str):
221
+ base = self.statistic_samples.calls[wrt]
222
+ else:
223
+ base = np.array([self.statistic_samples.calls[key][:, col]
224
+ for col, key in enumerate(wrt)]).T
207
225
  sign = 1 if self.statistic_samples.BiB else -1
208
226
  diff = dict()
209
227
  for k, v in self.statistic_samples.calls.items():
210
- if k == wrt_to:
228
+ if base.ndim == 1 and k == wrt:
211
229
  continue
212
230
  diff[k] = sign * (base - v)
213
231
  diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
214
- statistic=self.statistic,
215
- best=self.best[0])
232
+ statistic=self.statistic)
216
233
  diff_ins.sorting_func = self.sorting_func
217
234
  diff_ins.statistic_samples.calls = diff
218
- diff_ins.statistic_samples.info['best'] = self.best[0]
235
+ diff_ins.statistic_samples.info['best'] = self.best
236
+ diff_ins.best = self.best
219
237
  return diff_ins
220
238
 
221
239
  @property
222
240
  def best(self):
223
- """Best system"""
224
-
225
- try:
241
+ """System with best performance"""
242
+ if hasattr(self, '_best') and self._best is not None:
226
243
  return self._best
227
- except AttributeError:
228
- statistic = [(k, v) for k, v in self.statistic.items()]
229
- statistic = sorted(statistic,
230
- key=lambda x: self.sorting_func(x[1]),
231
- reverse=self.statistic_samples.BiB)
232
- self._best = statistic[0]
244
+ if not isinstance(self.statistic, dict):
245
+ key, value = list(self.statistic_samples.calls.items())[0]
246
+ if value.ndim == 1:
247
+ self._best = key
248
+ else:
249
+ self._best = np.array([key] * value.shape[1])
250
+ return self._best
251
+ BiB = True if self.statistic_samples.BiB else False
252
+ keys = np.array(list(self.statistic.keys()))
253
+ data = np.asanyarray([self.statistic[k]
254
+ for k in keys])
255
+ if isinstance(self.statistic[keys[0]], np.ndarray):
256
+ if BiB:
257
+ best = data.argmax(axis=0)
258
+ else:
259
+ best = data.argmin(axis=0)
260
+ else:
261
+ if BiB:
262
+ best = data.argmax()
263
+ else:
264
+ best = data.argmin()
265
+ self._best = keys[best]
233
266
  return self._best
234
267
 
268
+ @best.setter
269
+ def best(self, value):
270
+ self._best = value
271
+
235
272
  @property
236
273
  def sorting_func(self):
237
274
  """Rank systems when multiple performances are used"""
@@ -265,6 +302,8 @@ class Perf(object):
265
302
  for k, v in self.predictions.items()],
266
303
  key=lambda x: self.sorting_func(x[1]),
267
304
  reverse=self.statistic_samples.BiB)
305
+ if len(data) == 1:
306
+ return data[0][1]
268
307
  return dict(data)
269
308
 
270
309
  @property
@@ -287,9 +326,19 @@ class Perf(object):
287
326
  {'alg-1': 0.0, 'forest': 0.026945730782184187}
288
327
  """
289
328
 
290
- return SE(self.statistic_samples)
291
-
292
- def plot(self, **kwargs):
329
+ output = SE(self.statistic_samples)
330
+ if len(output) == 1:
331
+ return list(output.values())[0]
332
+ return output
333
+
334
+ def plot(self, value_name:str=None,
335
+ var_name:str='Performance',
336
+ alg_legend:str='Algorithm',
337
+ perf_names:list=None,
338
+ CI:float=0.05,
339
+ kind:str='point', linestyle:str='none',
340
+ col_wrap:int=3, capsize:float=0.2,
341
+ **kwargs):
293
342
  """plot with seaborn
294
343
 
295
344
  >>> from sklearn.svm import LinearSVC
@@ -308,13 +357,37 @@ class Perf(object):
308
357
  forest=ens.predict(X_val))
309
358
  >>> perf.plot()
310
359
  """
360
+ import seaborn as sns
311
361
  if self.score_func is not None:
312
362
  value_name = 'Score'
313
363
  else:
314
364
  value_name = 'Error'
315
- _ = dict(value_name=value_name)
316
- _.update(kwargs)
317
- return plot_performance(self.statistic_samples, **_)
365
+ df = self.dataframe(value_name=value_name, var_name=var_name,
366
+ alg_legend=alg_legend, perf_names=perf_names)
367
+ if var_name not in df.columns:
368
+ var_name = None
369
+ col_wrap = None
370
+ ci = lambda x: measurements.CI(x, alpha=CI)
371
+ f_grid = sns.catplot(df, x=value_name, errorbar=ci,
372
+ y=alg_legend, col=var_name,
373
+ kind=kind, linestyle=linestyle,
374
+ col_wrap=col_wrap, capsize=capsize, **kwargs)
375
+ return f_grid
376
+
377
+
378
+ def dataframe(self, value_name:str='Score',
379
+ var_name:str='Performance',
380
+ alg_legend:str='Algorithm',
381
+ perf_names:str=None):
382
+ """Dataframe"""
383
+ if perf_names is None and isinstance(self.best, np.ndarray):
384
+ func_name = self.statistic_func.__name__
385
+ perf_names = [f'{func_name}({i})'
386
+ for i, k in enumerate(self.best)]
387
+ return dataframe(self, value_name=value_name,
388
+ var_name=var_name,
389
+ alg_legend=alg_legend,
390
+ perf_names=perf_names)
318
391
 
319
392
  @property
320
393
  def n_jobs(self):
@@ -439,8 +512,8 @@ class Difference:
439
512
  """
440
513
 
441
514
  statistic_samples:StatisticSamples=None
442
- best:str=None
443
515
  statistic:dict=None
516
+ best:str=None
444
517
 
445
518
  @property
446
519
  def sorting_func(self):
@@ -453,11 +526,19 @@ class Difference:
453
526
 
454
527
  def __repr__(self):
455
528
  """p-value"""
456
- return f"<{self.__class__.__name__}>\n{self}"
529
+ return f"<{self.__class__.__name__}>\n{self}"
457
530
 
458
531
  def __str__(self):
459
532
  """p-value"""
460
- output = [f"difference p-values w.r.t {self.best}"]
533
+ if isinstance(self.best, str):
534
+ best = f' w.r.t {self.best}'
535
+ else:
536
+ best = ''
537
+ output = [f"difference p-values {best}"]
538
+ best = self.best
539
+ if isinstance(best, np.ndarray):
540
+ desc = ', '.join(best)
541
+ output.append(f'{desc} <= Best')
461
542
  for key, value in self.p_value().items():
462
543
  if isinstance(value, float):
463
544
  output.append(f'{value:0.4f} <= {key}')
@@ -465,8 +546,20 @@ class Difference:
465
546
  desc = [f'{v:0.4f}' for v in value]
466
547
  desc = ', '.join(desc)
467
548
  desc = f'{desc} <= {key}'
549
+ output.append(desc)
468
550
  return "\n".join(output)
469
551
 
552
+ def _delta_best(self):
553
+ """Compute multiple delta"""
554
+ if isinstance(self.best, str):
555
+ return self.statistic[self.best]
556
+ keys = np.unique(self.best)
557
+ statistic = np.array([self.statistic[k]
558
+ for k in keys])
559
+ m = {v: k for k, v in enumerate(keys)}
560
+ best = np.array([m[x] for x in self.best])
561
+ return statistic[best, np.arange(best.shape[0])]
562
+
470
563
  def p_value(self, right:bool=True):
471
564
  """Compute p_value of the differences
472
565
 
@@ -492,10 +585,10 @@ class Difference:
492
585
  """
493
586
  values = []
494
587
  sign = 1 if self.statistic_samples.BiB else -1
495
- ndim = self.statistic[self.best].ndim
588
+ delta_best = self._delta_best()
496
589
  for k, v in self.statistic_samples.calls.items():
497
- delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
498
- if ndim == 0:
590
+ delta = 2 * sign * (delta_best - self.statistic[k])
591
+ if not isinstance(delta_best, np.ndarray):
499
592
  if right:
500
593
  values.append((k, (v >= delta).mean()))
501
594
  else:
@@ -508,7 +601,28 @@ class Difference:
508
601
  values.sort(key=lambda x: self.sorting_func(x[1]))
509
602
  return dict(values)
510
603
 
511
- def plot(self, **kwargs):
604
+ def dataframe(self, value_name:str='Score',
605
+ var_name:str='Best',
606
+ alg_legend:str='Algorithm',
607
+ perf_names:str=None):
608
+ """Dataframe"""
609
+ if perf_names is None and isinstance(self.best, np.ndarray):
610
+ perf_names = [f'{alg}({k})'
611
+ for k, alg in enumerate(self.best)]
612
+ return dataframe(self, value_name=value_name,
613
+ var_name=var_name,
614
+ alg_legend=alg_legend,
615
+ perf_names=perf_names)
616
+
617
+ def plot(self, value_name:str='Difference',
618
+ var_name:str='Best',
619
+ alg_legend:str='Algorithm',
620
+ perf_names:list=None,
621
+ CI:float=0.05,
622
+ kind:str='point', linestyle:str='none',
623
+ col_wrap:int=3, capsize:float=0.2,
624
+ set_refline:bool=True,
625
+ **kwargs):
512
626
  """Plot
513
627
 
514
628
  >>> from sklearn.svm import LinearSVC
@@ -527,5 +641,21 @@ class Difference:
527
641
  >>> diff = perf.difference()
528
642
  >>> diff.plot()
529
643
  """
530
-
531
- return plot_difference(self.statistic_samples, **kwargs)
644
+ import seaborn as sns
645
+ df = self.dataframe(value_name=value_name,
646
+ var_name=var_name,
647
+ alg_legend=alg_legend, perf_names=perf_names)
648
+ title = var_name
649
+ if var_name not in df.columns:
650
+ var_name = None
651
+ col_wrap = None
652
+ ci = lambda x: measurements.CI(x, alpha=CI)
653
+ f_grid = sns.catplot(df, x=value_name, errorbar=ci,
654
+ y=alg_legend, col=var_name,
655
+ kind=kind, linestyle=linestyle,
656
+ col_wrap=col_wrap, capsize=capsize, **kwargs)
657
+ if set_refline:
658
+ f_grid.refline(x=0)
659
+ if isinstance(self.best, str):
660
+ f_grid.facet_axis(0, 0).set_title(f'{title} = {self.best}')
661
+ return f_grid
@@ -0,0 +1,310 @@
1
+ # Copyright 2025 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import numpy as np
16
+ from sklearn.base import clone
17
+ from sklearn.svm import LinearSVC
18
+ from sklearn.ensemble import RandomForestClassifier
19
+ from sklearn.naive_bayes import GaussianNB
20
+ from sklearn.datasets import load_iris, load_digits
21
+ from sklearn.model_selection import train_test_split
22
+ import pandas as pd
23
+ from CompStats.tests.test_performance import DATA
24
+
25
+
26
+ def test_Difference_dataframe():
27
+ """Test Difference dataframe"""
28
+ from CompStats.metrics import f1_score
29
+
30
+ X, y = load_digits(return_X_y=True)
31
+ _ = train_test_split(X, y, test_size=0.3)
32
+ X_train, X_val, y_train, y_val = _
33
+ ens = RandomForestClassifier().fit(X_train, y_train)
34
+ nb = GaussianNB().fit(X_train, y_train)
35
+ svm = LinearSVC().fit(X_train, y_train)
36
+ score = f1_score(y_val, ens.predict(X_val),
37
+ average=None,
38
+ num_samples=50)
39
+ score(nb.predict(X_val))
40
+ score(svm.predict(X_val))
41
+ diff = score.difference()
42
+ df = diff.dataframe()
43
+ assert 'Best' in df.columns
44
+ score = f1_score(y_val, ens.predict(X_val),
45
+ average='macro',
46
+ num_samples=50)
47
+ score(nb.predict(X_val))
48
+ score(svm.predict(X_val))
49
+ diff = score.difference()
50
+ df = diff.dataframe()
51
+ assert 'Best' not in df.columns
52
+
53
+
54
+ def test_Perf_dataframe():
55
+ """Test Perf dataframe"""
56
+ from CompStats.metrics import f1_score
57
+
58
+ X, y = load_digits(return_X_y=True)
59
+ _ = train_test_split(X, y, test_size=0.3)
60
+ X_train, X_val, y_train, y_val = _
61
+ ens = RandomForestClassifier().fit(X_train, y_train)
62
+ nb = GaussianNB().fit(X_train, y_train)
63
+ svm = LinearSVC().fit(X_train, y_train)
64
+ score = f1_score(y_val, ens.predict(X_val),
65
+ average=None,
66
+ num_samples=50)
67
+ df = score.dataframe()
68
+ score(nb.predict(X_val))
69
+ score(svm.predict(X_val))
70
+ df = score.dataframe()
71
+ assert 'Performance' in df.columns
72
+ score = f1_score(y_val, ens.predict(X_val),
73
+ average='macro',
74
+ num_samples=50)
75
+ score(nb.predict(X_val))
76
+ score(svm.predict(X_val))
77
+ df = score.dataframe()
78
+ assert 'Performance' not in df.columns
79
+
80
+
81
+ def test_Perf_plot_multi():
82
+ """Test Perf plot multiple"""
83
+ from CompStats.metrics import f1_score
84
+
85
+ X, y = load_digits(return_X_y=True)
86
+ _ = train_test_split(X, y, test_size=0.3)
87
+ X_train, X_val, y_train, y_val = _
88
+ ens = RandomForestClassifier().fit(X_train, y_train)
89
+ nb = GaussianNB().fit(X_train, y_train)
90
+ svm = LinearSVC().fit(X_train, y_train)
91
+ score = f1_score(y_val, ens.predict(X_val),
92
+ average=None,
93
+ num_samples=50)
94
+ score(nb.predict(X_val))
95
+ score(svm.predict(X_val))
96
+ f_grid = score.plot()
97
+ assert f_grid is not None
98
+
99
+ def test_Perf_statistic_one():
100
+ """Test Perf statistic one alg"""
101
+ from CompStats.metrics import f1_score
102
+
103
+ X, y = load_digits(return_X_y=True)
104
+ _ = train_test_split(X, y, test_size=0.3)
105
+ X_train, X_val, y_train, y_val = _
106
+ ens = RandomForestClassifier().fit(X_train, y_train)
107
+ nb = GaussianNB().fit(X_train, y_train)
108
+ svm = LinearSVC().fit(X_train, y_train)
109
+ score = f1_score(y_val, ens.predict(X_val),
110
+ average=None,
111
+ num_samples=50)
112
+ assert isinstance(score.statistic, np.ndarray)
113
+ assert isinstance(str(score), str)
114
+ score = f1_score(y_val, ens.predict(X_val),
115
+ average='macro',
116
+ num_samples=50)
117
+ assert isinstance(score.statistic, float)
118
+ assert isinstance(str(score), str)
119
+ assert isinstance(score.se, float)
120
+
121
+ def test_Perf_best():
122
+ """Test Perf best"""
123
+ from CompStats.metrics import f1_score
124
+
125
+ X, y = load_digits(return_X_y=True)
126
+ _ = train_test_split(X, y, test_size=0.3)
127
+ X_train, X_val, y_train, y_val = _
128
+ ens = RandomForestClassifier().fit(X_train, y_train)
129
+ nb = GaussianNB().fit(X_train, y_train)
130
+ svm = LinearSVC().fit(X_train, y_train)
131
+ score = f1_score(y_val, average=None,
132
+ num_samples=50)
133
+ score(ens.predict(X_val), name='forest')
134
+ score(nb.predict(X_val), name='NB')
135
+ score(svm.predict(X_val), name='svm')
136
+ assert isinstance(score.best, np.ndarray)
137
+ score = f1_score(y_val, average='macro',
138
+ num_samples=50)
139
+ score(ens.predict(X_val), name='forest')
140
+ score(nb.predict(X_val), name='NB')
141
+ score(svm.predict(X_val), name='svm')
142
+ assert isinstance(score.best, str)
143
+
144
+
145
+ def test_difference_best():
146
+ """Test multiple performance measures"""
147
+ from CompStats.metrics import f1_score
148
+
149
+ X, y = load_digits(return_X_y=True)
150
+ _ = train_test_split(X, y, test_size=0.3)
151
+ X_train, X_val, y_train, y_val = _
152
+ ens = RandomForestClassifier().fit(X_train, y_train)
153
+ nb = GaussianNB().fit(X_train, y_train)
154
+ svm = LinearSVC().fit(X_train, y_train)
155
+ score = f1_score(y_val, average=None,
156
+ num_samples=50)
157
+ score(ens.predict(X_val), name='forest')
158
+ score(nb.predict(X_val), name='NB')
159
+ score(svm.predict(X_val), name='svm')
160
+ diff = score.difference()
161
+ assert isinstance(diff.best, np.ndarray)
162
+ score = f1_score(y_val, average='macro',
163
+ num_samples=50)
164
+ score(ens.predict(X_val), name='forest')
165
+ score(nb.predict(X_val), name='NB')
166
+ score(svm.predict(X_val), name='svm')
167
+ diff = score.difference()
168
+ assert isinstance(diff.best, str)
169
+
170
+
171
+ def test_difference_str__():
172
+ """Test f1_score"""
173
+ from CompStats.metrics import f1_score
174
+
175
+ X, y = load_iris(return_X_y=True)
176
+ _ = train_test_split(X, y, test_size=0.3)
177
+ X_train, X_val, y_train, y_val = _
178
+ ens = RandomForestClassifier().fit(X_train, y_train)
179
+ nb = GaussianNB().fit(X_train, y_train)
180
+ perf = f1_score(y_val, nb.predict(X_val),
181
+ forest=ens.predict(X_val),
182
+ num_samples=50, average=None)
183
+ diff = perf.difference()
184
+ p_values = diff.p_value(right=False)
185
+ dd = list(p_values.values())[0]
186
+ assert isinstance(dd, np.ndarray)
187
+ for average in ['macro', None]:
188
+ perf = f1_score(y_val, nb.predict(X_val),
189
+ forest=ens.predict(X_val),
190
+ num_samples=50, average=average)
191
+ diff = perf.difference()
192
+ print(diff)
193
+
194
+
195
+ def test_Perf():
196
+ """Test perf"""
197
+ from CompStats.interface import Perf
198
+
199
+ X, y = load_iris(return_X_y=True)
200
+ _ = train_test_split(X, y, test_size=0.3)
201
+ X_train, X_val, y_train, y_val = _
202
+ m = LinearSVC().fit(X_train, y_train)
203
+ hy = m.predict(X_val)
204
+ ens = RandomForestClassifier().fit(X_train, y_train)
205
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
206
+ assert 'alg-1' in perf.predictions
207
+ assert 'forest' in perf.predictions
208
+ assert str(perf) is not None
209
+
210
+
211
+ def test_Perf_statistic():
212
+ """Test statistic"""
213
+ from CompStats.interface import Perf
214
+
215
+ X, y = load_iris(return_X_y=True)
216
+ _ = train_test_split(X, y, test_size=0.3)
217
+ X_train, X_val, y_train, y_val = _
218
+ ens = RandomForestClassifier().fit(X_train, y_train)
219
+ perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
220
+ perf(ens.predict(X_val))
221
+ assert 'forest' in perf.statistic
222
+
223
+
224
+ def test_Perf_plot():
225
+ """Test plot"""
226
+
227
+ from CompStats.interface import Perf
228
+
229
+ X, y = load_iris(return_X_y=True)
230
+ _ = train_test_split(X, y, test_size=0.3)
231
+ X_train, X_val, y_train, y_val = _
232
+ ens = RandomForestClassifier().fit(X_train, y_train)
233
+ perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
234
+ perf.plot()
235
+
236
+
237
+ def test_Perf_clone():
238
+ """Test Perf.clone"""
239
+ from CompStats.interface import Perf
240
+
241
+ X, y = load_iris(return_X_y=True)
242
+ _ = train_test_split(X, y, test_size=0.3)
243
+ X_train, X_val, y_train, y_val = _
244
+ ens = RandomForestClassifier().fit(X_train, y_train)
245
+ perf = Perf(y_val, forest=ens.predict(X_val), num_samples=50)
246
+ samples = perf.statistic_samples._samples
247
+ perf2 = clone(perf)
248
+ perf2.error_func = lambda y, hy: (y != hy).mean()
249
+ assert 'forest' in perf2.statistic_samples.calls
250
+ assert np.all(samples == perf2.statistic_samples._samples)
251
+
252
+
253
+ def test_Perf_difference():
254
+ """Test difference"""
255
+ from CompStats.interface import Perf, Difference
256
+
257
+ X, y = load_iris(return_X_y=True)
258
+ _ = train_test_split(X, y, test_size=0.3)
259
+ X_train, X_val, y_train, y_val = _
260
+ m = LinearSVC().fit(X_train, y_train)
261
+ hy = m.predict(X_val)
262
+ ens = RandomForestClassifier().fit(X_train, y_train)
263
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
264
+ diff = perf.difference()
265
+ assert isinstance(diff, Difference)
266
+ assert isinstance(str(diff), str)
267
+
268
+
269
+ def test_Difference_plot():
270
+ """Test difference plot"""
271
+ from CompStats.interface import Perf
272
+
273
+ X, y = load_iris(return_X_y=True)
274
+ _ = train_test_split(X, y, test_size=0.3)
275
+ X_train, X_val, y_train, y_val = _
276
+ m = LinearSVC().fit(X_train, y_train)
277
+ hy = m.predict(X_val)
278
+ ens = RandomForestClassifier().fit(X_train, y_train)
279
+ perf = Perf(y_val, hy, forest=ens.predict(X_val), num_samples=50)
280
+ diff = perf.difference()
281
+ diff.plot()
282
+
283
+
284
+ def test_Perf_input_dataframe():
285
+ """Test Perf with dataframe"""
286
+ from CompStats.interface import Perf
287
+
288
+ df = pd.read_csv(DATA)
289
+ perf = Perf(df, num_samples=50)
290
+ assert 'INGEOTEC' in perf.statistic
291
+
292
+
293
+ def test_Perf_call():
294
+ """Test Perf call"""
295
+ from CompStats.interface import Perf
296
+
297
+ X, y = load_iris(return_X_y=True)
298
+ _ = train_test_split(X, y, test_size=0.3)
299
+ X_train, X_val, y_train, y_val = _
300
+ m = LinearSVC().fit(X_train, y_train)
301
+ hy = m.predict(X_val)
302
+ ens = RandomForestClassifier().fit(X_train, y_train)
303
+ hy2 = ens.predict(X_val)
304
+ perf = Perf(y_val, num_samples=50)
305
+ for xx in [hy, hy2]:
306
+ _ = perf(xx)
307
+ print(_)
308
+ perf(hy, name='alg-2')
309
+ assert 'alg-2' not in perf._statistic_samples.calls
310
+ assert 'alg-1' in perf._statistic_samples.calls