CompStats 0.1.4__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. {compstats-0.1.4 → compstats-0.1.5}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.4 → compstats-0.1.5}/CompStats/interface.py +59 -9
  3. {compstats-0.1.4 → compstats-0.1.5}/CompStats/metrics.py +75 -29
  4. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_metrics.py +24 -0
  5. compstats-0.1.5/CompStats/utils.py +57 -0
  6. {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/PKG-INFO +1 -1
  7. {compstats-0.1.4 → compstats-0.1.5}/PKG-INFO +1 -1
  8. compstats-0.1.4/CompStats/utils.py +0 -55
  9. {compstats-0.1.4 → compstats-0.1.5}/CompStats/bootstrap.py +0 -0
  10. {compstats-0.1.4 → compstats-0.1.5}/CompStats/measurements.py +0 -0
  11. {compstats-0.1.4 → compstats-0.1.5}/CompStats/performance.py +0 -0
  12. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/__init__.py +0 -0
  13. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/data.csv +0 -0
  14. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_bootstrap.py +0 -0
  15. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_interface.py +0 -0
  16. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_measurements.py +0 -0
  17. {compstats-0.1.4 → compstats-0.1.5}/CompStats/tests/test_performance.py +0 -0
  18. {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/SOURCES.txt +0 -0
  19. {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/dependency_links.txt +0 -0
  20. {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/requires.txt +0 -0
  21. {compstats-0.1.4 → compstats-0.1.5}/CompStats.egg-info/top_level.txt +0 -0
  22. {compstats-0.1.4 → compstats-0.1.5}/LICENSE +0 -0
  23. {compstats-0.1.4 → compstats-0.1.5}/MANIFEST.in +0 -0
  24. {compstats-0.1.4 → compstats-0.1.5}/README.rst +0 -0
  25. {compstats-0.1.4 → compstats-0.1.5}/pyproject.toml +0 -0
  26. {compstats-0.1.4 → compstats-0.1.5}/setup.cfg +0 -0
  27. {compstats-0.1.4 → compstats-0.1.5}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.4'
14
+ __version__ = '0.1.5'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -107,6 +107,7 @@ class Perf(object):
107
107
  self.num_samples = num_samples
108
108
  self.n_jobs = n_jobs
109
109
  self.use_tqdm = use_tqdm
110
+ self.sorting_func = np.linalg.norm
110
111
  self._init()
111
112
 
112
113
  def _init(self):
@@ -139,11 +140,14 @@ class Perf(object):
139
140
  ins = klass(**params)
140
141
  ins.predictions = dict(self.predictions)
141
142
  ins._statistic_samples._samples = self.statistic_samples._samples
143
+ ins.sorting_func = self.sorting_func
142
144
  return ins
143
145
 
144
146
  def __repr__(self):
145
147
  """Prediction statistics with standard error in parenthesis"""
146
- return f"<{self.__class__.__name__}>\n{self}"
148
+ arg = 'score_func' if self.error_func is None else 'error_func'
149
+ func_name = self.statistic_func.__name__
150
+ return f"<{self.__class__.__name__}({arg}={func_name})>\n{self}"
147
151
 
148
152
  def __str__(self):
149
153
  """Prediction statistics with standard error in parenthesis"""
@@ -152,7 +156,14 @@ class Perf(object):
152
156
  output = ["Statistic with its standard error (se)"]
153
157
  output.append("statistic (se)")
154
158
  for key, value in self.statistic.items():
155
- output.append(f'{value:0.4f} ({se[key]:0.4f}) <= {key}')
159
+ if isinstance(value, float):
160
+ desc = f'{value:0.4f} ({se[key]:0.4f}) <= {key}'
161
+ else:
162
+ desc = [f'{v:0.4f} ({k:0.4f})'
163
+ for v, k in zip(value, se[key])]
164
+ desc = ', '.join(desc)
165
+ desc = f'{desc} <= {key}'
166
+ output.append(desc)
156
167
  return "\n".join(output)
157
168
 
158
169
  def __call__(self, y_pred, name=None):
@@ -202,6 +213,7 @@ class Perf(object):
202
213
  diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
203
214
  statistic=self.statistic,
204
215
  best=self.best[0])
216
+ diff_ins.sorting_func = self.sorting_func
205
217
  diff_ins.statistic_samples.calls = diff
206
218
  diff_ins.statistic_samples.info['best'] = self.best[0]
207
219
  return diff_ins
@@ -214,10 +226,20 @@ class Perf(object):
214
226
  return self._best
215
227
  except AttributeError:
216
228
  statistic = [(k, v) for k, v in self.statistic.items()]
217
- statistic = sorted(statistic, key=lambda x: x[1],
229
+ statistic = sorted(statistic,
230
+ key=lambda x: self.sorting_func(x[1]),
218
231
  reverse=self.statistic_samples.BiB)
219
232
  self._best = statistic[0]
220
233
  return self._best
234
+
235
+ @property
236
+ def sorting_func(self):
237
+ """Rank systems when multiple performances are used"""
238
+ return self._sorting_func
239
+
240
+ @sorting_func.setter
241
+ def sorting_func(self, value):
242
+ self._sorting_func = value
221
243
 
222
244
  @property
223
245
  def statistic(self):
@@ -241,7 +263,8 @@ class Perf(object):
241
263
 
242
264
  data = sorted([(k, self.statistic_func(self.y_true, v))
243
265
  for k, v in self.predictions.items()],
244
- key=lambda x: x[1], reverse=self.statistic_samples.BiB)
266
+ key=lambda x: self.sorting_func(x[1]),
267
+ reverse=self.statistic_samples.BiB)
245
268
  return dict(data)
246
269
 
247
270
  @property
@@ -419,6 +442,15 @@ class Difference:
419
442
  best:str=None
420
443
  statistic:dict=None
421
444
 
445
+ @property
446
+ def sorting_func(self):
447
+ """Rank systems when multiple performances are used"""
448
+ return self._sorting_func
449
+
450
+ @sorting_func.setter
451
+ def sorting_func(self, value):
452
+ self._sorting_func = value
453
+
422
454
  def __repr__(self):
423
455
  """p-value"""
424
456
  return f"<{self.__class__.__name__}>\n{self}"
@@ -426,12 +458,20 @@ class Difference:
426
458
  def __str__(self):
427
459
  """p-value"""
428
460
  output = [f"difference p-values w.r.t {self.best}"]
429
- for k, v in self.p_value().items():
430
- output.append(f'{v:0.4f} <= {k}')
461
+ for key, value in self.p_value().items():
462
+ if isinstance(value, float):
463
+ output.append(f'{value:0.4f} <= {key}')
464
+ else:
465
+ desc = [f'{v:0.4f}' for v in value]
466
+ desc = ', '.join(desc)
467
+ desc = f'{desc} <= {key}'
431
468
  return "\n".join(output)
432
469
 
433
- def p_value(self):
470
+ def p_value(self, right:bool=True):
434
471
  """Compute p_value of the differences
472
+
473
+ :param right: Estimate the p-value using :math:`\\text{sample} \\geq 2\\delta`
474
+ :type right: bool
435
475
 
436
476
  >>> from sklearn.svm import LinearSVC
437
477
  >>> from sklearn.ensemble import RandomForestClassifier
@@ -452,10 +492,20 @@ class Difference:
452
492
  """
453
493
  values = []
454
494
  sign = 1 if self.statistic_samples.BiB else -1
495
+ ndim = self.statistic[self.best].ndim
455
496
  for k, v in self.statistic_samples.calls.items():
456
497
  delta = 2 * sign * (self.statistic[self.best] - self.statistic[k])
457
- values.append((k, (v > delta).mean()))
458
- values.sort(key=lambda x: x[1])
498
+ if ndim == 0:
499
+ if right:
500
+ values.append((k, (v >= delta).mean()))
501
+ else:
502
+ values.append((k, (v <= 0).mean()))
503
+ else:
504
+ if right:
505
+ values.append((k, (v >= delta).mean(axis=0)))
506
+ else:
507
+ values.append((k, (v <= 0).mean(axis=0)))
508
+ values.sort(key=lambda x: self.sorting_func(x[1]))
459
509
  return dict(values)
460
510
 
461
511
  def plot(self, **kwargs):
@@ -11,15 +11,18 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
+ from functools import wraps
14
15
  from sklearn import metrics
15
16
  from CompStats.interface import Perf
16
- from CompStats.utils import perf_docs
17
+ from CompStats.utils import metrics_docs
18
+
17
19
 
18
20
  ########################################################
19
21
  #################### Classification ####################
20
22
  ########################################################
21
23
 
22
- @perf_docs
24
+
25
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
23
26
  def accuracy_score(y_true, *y_pred,
24
27
  normalize=True, sample_weight=None,
25
28
  num_samples: int=500,
@@ -28,6 +31,7 @@ def accuracy_score(y_true, *y_pred,
28
31
  **kwargs):
29
32
  """accuracy_score"""
30
33
 
34
+ @wraps(metrics.accuracy_score)
31
35
  def inner(y, hy):
32
36
  return metrics.accuracy_score(y, hy,
33
37
  normalize=normalize,
@@ -38,7 +42,7 @@ def accuracy_score(y_true, *y_pred,
38
42
  **kwargs)
39
43
 
40
44
 
41
- @perf_docs
45
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
42
46
  def balanced_accuracy_score(y_true, *y_pred,
43
47
  sample_weight=None, adjusted=False,
44
48
  num_samples: int=500,
@@ -47,6 +51,7 @@ def balanced_accuracy_score(y_true, *y_pred,
47
51
  **kwargs):
48
52
  """balanced_accuracy_score"""
49
53
 
54
+ @wraps(metrics.balanced_accuracy_score)
50
55
  def inner(y, hy):
51
56
  return metrics.balanced_accuracy_score(y, hy,
52
57
  adjusted=adjusted,
@@ -57,7 +62,7 @@ def balanced_accuracy_score(y_true, *y_pred,
57
62
  **kwargs)
58
63
 
59
64
 
60
- @perf_docs
65
+ @metrics_docs(hy_name='y_score', attr_name='score_func')
61
66
  def top_k_accuracy_score(y_true, *y_score, k=2,
62
67
  normalize=True, sample_weight=None,
63
68
  labels=None,
@@ -67,6 +72,7 @@ def top_k_accuracy_score(y_true, *y_score, k=2,
67
72
  **kwargs):
68
73
  """top_k_accuracy_score"""
69
74
 
75
+ @wraps(metrics.top_k_accuracy_score)
70
76
  def inner(y, hy):
71
77
  return metrics.top_k_accuracy_score(y, hy, k=k,
72
78
  normalize=normalize, sample_weight=sample_weight,
@@ -77,7 +83,7 @@ def top_k_accuracy_score(y_true, *y_score, k=2,
77
83
  **kwargs)
78
84
 
79
85
 
80
- @perf_docs
86
+ @metrics_docs(hy_name='y_score', attr_name='score_func')
81
87
  def average_precision_score(y_true, *y_score,
82
88
  average='macro',
83
89
  sample_weight=None,
@@ -87,6 +93,7 @@ def average_precision_score(y_true, *y_score,
87
93
  **kwargs):
88
94
  """average_precision_score"""
89
95
 
96
+ @wraps(metrics.average_precision_score)
90
97
  def inner(y, hy):
91
98
  return metrics.average_precision_score(y, hy,
92
99
  average=average,
@@ -97,7 +104,7 @@ def average_precision_score(y_true, *y_score,
97
104
  **kwargs)
98
105
 
99
106
 
100
- @perf_docs
107
+ @metrics_docs(hy_name='y_proba', attr_name='error_func')
101
108
  def brier_score_loss(y_true, *y_proba,
102
109
  sample_weight=None,
103
110
  pos_label=None,
@@ -108,6 +115,7 @@ def brier_score_loss(y_true, *y_proba,
108
115
  ):
109
116
  """brier_score_loss"""
110
117
 
118
+ @wraps(metrics.brier_score_loss)
111
119
  def inner(y, hy):
112
120
  return metrics.brier_score_loss(y, hy,
113
121
  sample_weight=sample_weight,
@@ -116,9 +124,9 @@ def brier_score_loss(y_true, *y_proba,
116
124
  num_samples=num_samples, n_jobs=n_jobs,
117
125
  use_tqdm=use_tqdm,
118
126
  **kwargs)
119
-
120
127
 
121
- @perf_docs
128
+
129
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
122
130
  def f1_score(y_true, *y_pred, labels=None, pos_label=1,
123
131
  average='binary', sample_weight=None,
124
132
  zero_division='warn', num_samples: int=500,
@@ -126,6 +134,7 @@ def f1_score(y_true, *y_pred, labels=None, pos_label=1,
126
134
  **kwargs):
127
135
  """f1_score"""
128
136
 
137
+ @wraps(metrics.f1_score)
129
138
  def inner(y, hy):
130
139
  return metrics.f1_score(y, hy, labels=labels,
131
140
  pos_label=pos_label,
@@ -138,7 +147,7 @@ def f1_score(y_true, *y_pred, labels=None, pos_label=1,
138
147
  **kwargs)
139
148
 
140
149
 
141
- @perf_docs
150
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
142
151
  def log_loss(y_true, *y_pred,
143
152
  normalize=True,
144
153
  sample_weight=None,
@@ -148,6 +157,8 @@ def log_loss(y_true, *y_pred,
148
157
  use_tqdm=True,
149
158
  **kwargs):
150
159
  """log_loss"""
160
+
161
+ @wraps(metrics.log_loss)
151
162
  def inner(y, hy):
152
163
  return metrics.log_loss(y, hy, normalize=normalize,
153
164
  sample_weight=sample_weight,
@@ -158,7 +169,7 @@ def log_loss(y_true, *y_pred,
158
169
  **kwargs)
159
170
 
160
171
 
161
- @perf_docs
172
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
162
173
  def precision_score(y_true,
163
174
  *y_pred,
164
175
  labels=None,
@@ -171,6 +182,8 @@ def precision_score(y_true,
171
182
  use_tqdm=True,
172
183
  **kwargs):
173
184
  """precision_score"""
185
+
186
+ @wraps(metrics.precision_score)
174
187
  def inner(y, hy):
175
188
  return metrics.precision_score(y, hy,
176
189
  labels=labels,
@@ -184,7 +197,7 @@ def precision_score(y_true,
184
197
  **kwargs)
185
198
 
186
199
 
187
- @perf_docs
200
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
188
201
  def recall_score(y_true,
189
202
  *y_pred,
190
203
  labels=None,
@@ -197,6 +210,8 @@ def recall_score(y_true,
197
210
  use_tqdm=True,
198
211
  **kwargs):
199
212
  """recall_score"""
213
+
214
+ @wraps(metrics.recall_score)
200
215
  def inner(y, hy):
201
216
  return metrics.recall_score(y, hy,
202
217
  labels=labels,
@@ -210,7 +225,7 @@ def recall_score(y_true,
210
225
  **kwargs)
211
226
 
212
227
 
213
- @perf_docs
228
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
214
229
  def jaccard_score(y_true,
215
230
  *y_pred,
216
231
  labels=None,
@@ -223,6 +238,8 @@ def jaccard_score(y_true,
223
238
  use_tqdm=True,
224
239
  **kwargs):
225
240
  """jaccard_score"""
241
+
242
+ @wraps(metrics.jaccard_score)
226
243
  def inner(y, hy):
227
244
  return metrics.jaccard_score(y, hy,
228
245
  labels=labels,
@@ -236,7 +253,7 @@ def jaccard_score(y_true,
236
253
  **kwargs)
237
254
 
238
255
 
239
- @perf_docs
256
+ @metrics_docs(hy_name='y_score', attr_name='score_func')
240
257
  def roc_auc_score(y_true,
241
258
  *y_score,
242
259
  average='macro',
@@ -249,6 +266,8 @@ def roc_auc_score(y_true,
249
266
  use_tqdm=True,
250
267
  **kwargs):
251
268
  """roc_auc_score"""
269
+
270
+ @wraps(metrics.roc_auc_score)
252
271
  def inner(y, hy):
253
272
  return metrics.roc_auc_score(y, hy,
254
273
  average=average,
@@ -262,7 +281,7 @@ def roc_auc_score(y_true,
262
281
  **kwargs)
263
282
 
264
283
 
265
- @perf_docs
284
+ @metrics_docs(hy_name='y_proba', attr_name='score_func')
266
285
  def d2_log_loss_score(y_true, *y_proba,
267
286
  sample_weight=None,
268
287
  labels=None,
@@ -272,6 +291,7 @@ def d2_log_loss_score(y_true, *y_proba,
272
291
  **kwargs):
273
292
  """d2_log_loss_score"""
274
293
 
294
+ @wraps(metrics.d2_log_loss_score)
275
295
  def inner(y, hy):
276
296
  return metrics.d2_log_loss_score(y, hy,
277
297
  sample_weight=sample_weight,
@@ -287,7 +307,7 @@ def d2_log_loss_score(y_true, *y_proba,
287
307
  ########################################################
288
308
 
289
309
 
290
- @perf_docs
310
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
291
311
  def explained_variance_score(y_true,
292
312
  *y_pred,
293
313
  sample_weight=None,
@@ -298,6 +318,8 @@ def explained_variance_score(y_true,
298
318
  use_tqdm=True,
299
319
  **kwargs):
300
320
  """explained_variance_score"""
321
+
322
+ @wraps(metrics.explained_variance_score)
301
323
  def inner(y, hy):
302
324
  return metrics.explained_variance_score(y, hy,
303
325
  sample_weight=sample_weight,
@@ -309,13 +331,15 @@ def explained_variance_score(y_true,
309
331
  **kwargs)
310
332
 
311
333
 
312
- @perf_docs
334
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
313
335
  def max_error(y_true, *y_pred,
314
336
  num_samples: int=500,
315
337
  n_jobs: int=-1,
316
338
  use_tqdm=True,
317
339
  **kwargs):
318
340
  """max_error"""
341
+
342
+ @wraps(metrics.max_error)
319
343
  def inner(y, hy):
320
344
  return metrics.max_error(y, hy)
321
345
  return Perf(y_true, *y_pred, score_func=None, error_func=inner,
@@ -324,7 +348,7 @@ def max_error(y_true, *y_pred,
324
348
  **kwargs)
325
349
 
326
350
 
327
- @perf_docs
351
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
328
352
  def mean_absolute_error(y_true,
329
353
  *y_pred,
330
354
  sample_weight=None,
@@ -334,6 +358,8 @@ def mean_absolute_error(y_true,
334
358
  use_tqdm=True,
335
359
  **kwargs):
336
360
  """mean_absolute_error"""
361
+
362
+ @wraps(metrics.mean_absolute_error)
337
363
  def inner(y, hy):
338
364
  return metrics.mean_absolute_error(y, hy,
339
365
  sample_weight=sample_weight,
@@ -345,7 +371,7 @@ def mean_absolute_error(y_true,
345
371
  **kwargs)
346
372
 
347
373
 
348
- @perf_docs
374
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
349
375
  def mean_squared_error(y_true,
350
376
  *y_pred,
351
377
  sample_weight=None,
@@ -355,6 +381,8 @@ def mean_squared_error(y_true,
355
381
  use_tqdm=True,
356
382
  **kwargs):
357
383
  """mean_squared_error"""
384
+
385
+ @wraps(metrics.mean_squared_error)
358
386
  def inner(y, hy):
359
387
  return metrics.mean_squared_error(y, hy,
360
388
  sample_weight=sample_weight,
@@ -366,7 +394,7 @@ def mean_squared_error(y_true,
366
394
  **kwargs)
367
395
 
368
396
 
369
- @perf_docs
397
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
370
398
  def root_mean_squared_error(y_true,
371
399
  *y_pred,
372
400
  sample_weight=None,
@@ -376,6 +404,8 @@ def root_mean_squared_error(y_true,
376
404
  use_tqdm=True,
377
405
  **kwargs):
378
406
  """root_mean_squared_error"""
407
+
408
+ @wraps(metrics.root_mean_squared_error)
379
409
  def inner(y, hy):
380
410
  return metrics.root_mean_squared_error(y, hy,
381
411
  sample_weight=sample_weight,
@@ -387,7 +417,7 @@ def root_mean_squared_error(y_true,
387
417
  **kwargs)
388
418
 
389
419
 
390
- @perf_docs
420
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
391
421
  def mean_squared_log_error(y_true,
392
422
  *y_pred,
393
423
  sample_weight=None,
@@ -397,6 +427,8 @@ def mean_squared_log_error(y_true,
397
427
  use_tqdm=True,
398
428
  **kwargs):
399
429
  """mean_squared_log_error"""
430
+
431
+ @wraps(metrics.mean_squared_log_error)
400
432
  def inner(y, hy):
401
433
  return metrics.mean_squared_log_error(y, hy,
402
434
  sample_weight=sample_weight,
@@ -408,7 +440,7 @@ def mean_squared_log_error(y_true,
408
440
  **kwargs)
409
441
 
410
442
 
411
- @perf_docs
443
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
412
444
  def root_mean_squared_log_error(y_true,
413
445
  *y_pred,
414
446
  sample_weight=None,
@@ -418,6 +450,8 @@ def root_mean_squared_log_error(y_true,
418
450
  use_tqdm=True,
419
451
  **kwargs):
420
452
  """root_mean_squared_log_error"""
453
+
454
+ @wraps(metrics.root_mean_squared_log_error)
421
455
  def inner(y, hy):
422
456
  return metrics.root_mean_squared_log_error(y, hy,
423
457
  sample_weight=sample_weight,
@@ -429,7 +463,7 @@ def root_mean_squared_log_error(y_true,
429
463
  **kwargs)
430
464
 
431
465
 
432
- @perf_docs
466
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
433
467
  def median_absolute_error(y_true,
434
468
  *y_pred,
435
469
  sample_weight=None,
@@ -439,10 +473,12 @@ def median_absolute_error(y_true,
439
473
  use_tqdm=True,
440
474
  **kwargs):
441
475
  """median_absolute_error"""
476
+
477
+ @wraps(metrics.median_absolute_error)
442
478
  def inner(y, hy):
443
479
  return metrics.median_absolute_error(y, hy,
444
- sample_weight=sample_weight,
445
- multioutput=multioutput)
480
+ sample_weight=sample_weight,
481
+ multioutput=multioutput)
446
482
 
447
483
  return Perf(y_true, *y_pred, score_func=None, error_func=inner,
448
484
  num_samples=num_samples, n_jobs=n_jobs,
@@ -450,7 +486,7 @@ def median_absolute_error(y_true,
450
486
  **kwargs)
451
487
 
452
488
 
453
- @perf_docs
489
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
454
490
  def r2_score(y_true,
455
491
  *y_pred,
456
492
  sample_weight=None,
@@ -461,6 +497,8 @@ def r2_score(y_true,
461
497
  use_tqdm=True,
462
498
  **kwargs):
463
499
  """r2_score"""
500
+
501
+ @wraps(metrics.r2_score)
464
502
  def inner(y, hy):
465
503
  return metrics.r2_score(y, hy,
466
504
  sample_weight=sample_weight,
@@ -473,7 +511,7 @@ def r2_score(y_true,
473
511
  **kwargs)
474
512
 
475
513
 
476
- @perf_docs
514
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
477
515
  def mean_poisson_deviance(y_true,
478
516
  *y_pred,
479
517
  sample_weight=None,
@@ -482,6 +520,8 @@ def mean_poisson_deviance(y_true,
482
520
  use_tqdm=True,
483
521
  **kwargs):
484
522
  """mean_poisson_deviance"""
523
+
524
+ @wraps(metrics.mean_poisson_deviance)
485
525
  def inner(y, hy):
486
526
  return metrics.mean_poisson_deviance(y, hy,
487
527
  sample_weight=sample_weight)
@@ -492,7 +532,7 @@ def mean_poisson_deviance(y_true,
492
532
  **kwargs)
493
533
 
494
534
 
495
- @perf_docs
535
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
496
536
  def mean_gamma_deviance(y_true,
497
537
  *y_pred,
498
538
  sample_weight=None,
@@ -501,6 +541,8 @@ def mean_gamma_deviance(y_true,
501
541
  use_tqdm=True,
502
542
  **kwargs):
503
543
  """mean_gamma_deviance"""
544
+
545
+ @wraps(metrics.mean_gamma_deviance)
504
546
  def inner(y, hy):
505
547
  return metrics.mean_gamma_deviance(y, hy,
506
548
  sample_weight=sample_weight)
@@ -511,7 +553,7 @@ def mean_gamma_deviance(y_true,
511
553
  **kwargs)
512
554
 
513
555
 
514
- @perf_docs
556
+ @metrics_docs(hy_name='y_pred', attr_name='error_func')
515
557
  def mean_absolute_percentage_error(y_true,
516
558
  *y_pred,
517
559
  sample_weight=None,
@@ -521,6 +563,8 @@ def mean_absolute_percentage_error(y_true,
521
563
  use_tqdm=True,
522
564
  **kwargs):
523
565
  """mean_absolute_percentage_error"""
566
+
567
+ @wraps(metrics.mean_absolute_percentage_error)
524
568
  def inner(y, hy):
525
569
  return metrics.mean_absolute_percentage_error(y, hy,
526
570
  sample_weight=sample_weight,
@@ -532,7 +576,7 @@ def mean_absolute_percentage_error(y_true,
532
576
  **kwargs)
533
577
 
534
578
 
535
- @perf_docs
579
+ @metrics_docs(hy_name='y_pred', attr_name='score_func')
536
580
  def d2_absolute_error_score(y_true,
537
581
  *y_pred,
538
582
  sample_weight=None,
@@ -542,6 +586,8 @@ def d2_absolute_error_score(y_true,
542
586
  use_tqdm=True,
543
587
  **kwargs):
544
588
  """d2_absolute_error_score"""
589
+
590
+ @wraps(metrics.d2_absolute_error_score)
545
591
  def inner(y, hy):
546
592
  return metrics.d2_absolute_error_score(y, hy,
547
593
  sample_weight=sample_weight,
@@ -11,6 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
+ import numpy as np
14
15
  from sklearn.svm import LinearSVC
15
16
  from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor
16
17
  from sklearn.naive_bayes import GaussianNB
@@ -19,6 +20,24 @@ from sklearn.model_selection import train_test_split
19
20
  from sklearn import metrics
20
21
 
21
22
 
23
+ def test_difference_f1_score():
24
+ """Test f1_score"""
25
+ from CompStats.metrics import f1_score
26
+
27
+ X, y = load_iris(return_X_y=True)
28
+ _ = train_test_split(X, y, test_size=0.3)
29
+ X_train, X_val, y_train, y_val = _
30
+ ens = RandomForestClassifier().fit(X_train, y_train)
31
+ nb = GaussianNB().fit(X_train, y_train)
32
+ perf = f1_score(y_val, nb.predict(X_val),
33
+ forest=ens.predict(X_val),
34
+ num_samples=50, average=None)
35
+ diff = perf.difference()
36
+ p_values = diff.p_value(right=False)
37
+ dd = list(p_values.values())[0]
38
+ assert isinstance(dd, np.ndarray)
39
+ print(diff)
40
+
22
41
  def test_f1_score():
23
42
  """Test f1_score"""
24
43
  from CompStats.metrics import f1_score
@@ -33,6 +52,11 @@ def test_f1_score():
33
52
  assert 'forest' in perf.statistic
34
53
  _ = metrics.f1_score(y_val, hy, average='macro')
35
54
  assert _ == perf.statistic['forest']
55
+ perf = f1_score(y_val, hy, average=None)
56
+ assert str(perf) is not None
57
+ nb = GaussianNB().fit(X_train, y_train)
58
+ perf(nb.predict(X_val))
59
+ assert str(perf) is not None
36
60
 
37
61
 
38
62
  def test_accuracy_score():
@@ -0,0 +1,57 @@
1
+ # Copyright 2024 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from functools import wraps
15
+ try:
16
+ USE_TQDM = True
17
+ from tqdm import tqdm
18
+ except ImportError:
19
+ USE_TQDM = False
20
+
21
+
22
+ def progress_bar(arg, use_tqdm: bool=True, **kwargs):
23
+ """Progress bar using tqdm"""
24
+ if not USE_TQDM or not use_tqdm:
25
+ return arg
26
+ return tqdm(arg, **kwargs)
27
+
28
+
29
+ def metrics_docs(hy_name='y_pred', attr_name='score_func'):
30
+ """Decorator to set docs"""
31
+
32
+ def perf_docs(func):
33
+ """Decorator to Perf to write :py:class:`~sklearn.metrics` documentation"""
34
+
35
+ func.__doc__ = f""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.{func.__name__}` as :py:attr:`{attr_name}.` The parameters not described can be found in :py:func:`~sklearn.metrics.{func.__name__}`.
36
+
37
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
38
+ :type y_true: numpy.ndarray or pandas.DataFrame
39
+ :param {hy_name}: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
40
+ :type {hy_name}: numpy.ndarray
41
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
42
+ :type kwargs: numpy.ndarray
43
+ :param num_samples: Number of bootstrap samples, default=500.
44
+ :type num_samples: int
45
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
46
+ :type n_jobs: int
47
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
48
+ :type use_tqdm: bool
49
+
50
+ """ + func.__doc__
51
+
52
+ @wraps(func)
53
+ def inner(*args, **kwargs):
54
+ return func(*args, **kwargs)
55
+
56
+ return inner
57
+ return perf_docs
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.2
2
2
  Name: CompStats
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -1,55 +0,0 @@
1
- # Copyright 2024 Sergio Nava Muñoz and Mario Graff Guerrero
2
-
3
- # Licensed under the Apache License, Version 2.0 (the "License");
4
- # you may not use this file except in compliance with the License.
5
- # You may obtain a copy of the License at
6
-
7
- # http://www.apache.org/licenses/LICENSE-2.0
8
-
9
- # Unless required by applicable law or agreed to in writing, software
10
- # distributed under the License is distributed on an "AS IS" BASIS,
11
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
- # See the License for the specific language governing permissions and
13
- # limitations under the License.
14
- try:
15
- USE_TQDM = True
16
- from tqdm import tqdm
17
- except ImportError:
18
- USE_TQDM = False
19
-
20
-
21
- def progress_bar(arg, use_tqdm: bool=True, **kwargs):
22
- """Progress bar using tqdm"""
23
- if not USE_TQDM or not use_tqdm:
24
- return arg
25
- return tqdm(arg, **kwargs)
26
-
27
-
28
- from functools import wraps
29
-
30
-
31
- def perf_docs(func):
32
- """Decorator to Perf with any write :py:class:`~sklearn.metrics` documentation
33
- """
34
-
35
- func.__doc__ = f""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.{func.__name__}` as :py:attr:`score_func.` The parameters not described can be found in :py:func:`~sklearn.metrics.{func.__name__}`.
36
-
37
- :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
38
- :type y_true: numpy.ndarray or pandas.DataFrame
39
- :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
40
- :type y_pred: numpy.ndarray
41
- :param kwargs: Predictions, the algorithms will be identified using the keyword
42
- :type kwargs: numpy.ndarray
43
- :param num_samples: Number of bootstrap samples, default=500.
44
- :type num_samples: int
45
- :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
46
- :type n_jobs: int
47
- :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
48
- :type use_tqdm: bool
49
- """ + func.__doc__
50
-
51
- @wraps(func)
52
- def inner(*args, **kwargs):
53
- return func(*args, **kwargs)
54
-
55
- return inner
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes