CompStats 0.1.12__tar.gz → 0.1.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {compstats-0.1.12 → compstats-0.1.13}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.12 → compstats-0.1.13}/CompStats/metrics.py +110 -1
  3. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_metrics.py +67 -1
  4. {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/PKG-INFO +1 -1
  5. {compstats-0.1.12 → compstats-0.1.13}/PKG-INFO +1 -1
  6. {compstats-0.1.12 → compstats-0.1.13}/CompStats/bootstrap.py +0 -0
  7. {compstats-0.1.12 → compstats-0.1.13}/CompStats/interface.py +0 -0
  8. {compstats-0.1.12 → compstats-0.1.13}/CompStats/measurements.py +0 -0
  9. {compstats-0.1.12 → compstats-0.1.13}/CompStats/performance.py +0 -0
  10. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/__init__.py +0 -0
  11. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/data.csv +0 -0
  12. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_bootstrap.py +0 -0
  13. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_interface.py +0 -0
  14. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_measurements.py +0 -0
  15. {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_performance.py +0 -0
  16. {compstats-0.1.12 → compstats-0.1.13}/CompStats/utils.py +0 -0
  17. {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/SOURCES.txt +0 -0
  18. {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/dependency_links.txt +0 -0
  19. {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/requires.txt +0 -0
  20. {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/top_level.txt +0 -0
  21. {compstats-0.1.12 → compstats-0.1.13}/LICENSE +0 -0
  22. {compstats-0.1.12 → compstats-0.1.13}/MANIFEST.in +0 -0
  23. {compstats-0.1.12 → compstats-0.1.13}/README.rst +0 -0
  24. {compstats-0.1.12 → compstats-0.1.13}/pyproject.toml +0 -0
  25. {compstats-0.1.12 → compstats-0.1.13}/setup.cfg +0 -0
  26. {compstats-0.1.12 → compstats-0.1.13}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.12'
14
+ __version__ = '0.1.13'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -13,6 +13,7 @@
13
13
  # limitations under the License.
14
14
  from functools import wraps
15
15
  from sklearn import metrics
16
+ from scipy import stats
16
17
  from CompStats.interface import Perf
17
18
  from CompStats.utils import metrics_docs
18
19
 
@@ -302,6 +303,81 @@ def d2_log_loss_score(y_true, *y_proba,
302
303
  **kwargs)
303
304
 
304
305
 
306
+ def macro_f1(y_true, *y_pred, labels=None,
307
+ sample_weight=None, zero_division='warn',
308
+ num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
309
+ **kwargs):
310
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_f1`
311
+
312
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
313
+ :type y_true: numpy.ndarray or pandas.DataFrame
314
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
315
+ :type y_pred: numpy.ndarray
316
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
317
+ :type kwargs: numpy.ndarray
318
+ :param num_samples: Number of bootstrap samples, default=500.
319
+ :type num_samples: int
320
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
321
+ :type n_jobs: int
322
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
323
+ :type use_tqdm: bool
324
+ """
325
+ return f1_score(y_true, *y_pred, labels=labels, average='macro',
326
+ sample_weight=sample_weight, zero_division=zero_division,
327
+ num_samples=num_samples, n_jobs=n_jobs,
328
+ use_tqdm=use_tqdm, **kwargs)
329
+
330
+
331
+ def macro_recall(y_true, *y_pred, labels=None,
332
+ sample_weight=None, zero_division='warn',
333
+ num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
334
+ **kwargs):
335
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_recall`
336
+
337
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
338
+ :type y_true: numpy.ndarray or pandas.DataFrame
339
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
340
+ :type y_pred: numpy.ndarray
341
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
342
+ :type kwargs: numpy.ndarray
343
+ :param num_samples: Number of bootstrap samples, default=500.
344
+ :type num_samples: int
345
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
346
+ :type n_jobs: int
347
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
348
+ :type use_tqdm: bool
349
+ """
350
+ return recall_score(y_true, *y_pred, labels=labels, average='macro',
351
+ sample_weight=sample_weight, zero_division=zero_division,
352
+ num_samples=num_samples, n_jobs=n_jobs,
353
+ use_tqdm=use_tqdm, **kwargs)
354
+
355
+
356
+ def macro_precision(y_true, *y_pred, labels=None,
357
+ sample_weight=None, zero_division='warn',
358
+ num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
359
+ **kwargs):
360
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_precision`
361
+
362
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
363
+ :type y_true: numpy.ndarray or pandas.DataFrame
364
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
365
+ :type y_pred: numpy.ndarray
366
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
367
+ :type kwargs: numpy.ndarray
368
+ :param num_samples: Number of bootstrap samples, default=500.
369
+ :type num_samples: int
370
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
371
+ :type n_jobs: int
372
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
373
+ :type use_tqdm: bool
374
+ """
375
+ return precision_score(y_true, *y_pred, labels=labels, average='macro',
376
+ sample_weight=sample_weight, zero_division=zero_division,
377
+ num_samples=num_samples, n_jobs=n_jobs,
378
+ use_tqdm=use_tqdm, **kwargs)
379
+
380
+
305
381
  ########################################################
306
382
  #################### Regression ########################
307
383
  ########################################################
@@ -576,7 +652,6 @@ def mean_absolute_percentage_error(y_true,
576
652
  **kwargs)
577
653
 
578
654
 
579
- @metrics_docs(hy_name='y_pred', attr_name='score_func')
580
655
  def d2_absolute_error_score(y_true,
581
656
  *y_pred,
582
657
  sample_weight=None,
@@ -597,3 +672,37 @@ def d2_absolute_error_score(y_true,
597
672
  num_samples=num_samples, n_jobs=n_jobs,
598
673
  use_tqdm=use_tqdm,
599
674
  **kwargs)
675
+
676
+
677
+ def pearsonr(y_true, *y_pred,
678
+ alternative='two-sided', method=None,
679
+ num_samples: int=500,
680
+ n_jobs: int=-1,
681
+ use_tqdm=True,
682
+ **kwargs):
683
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~scipy.stats.pearsonr` as :py:attr:`score_func.`
684
+
685
+ :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
686
+ :type y_true: numpy.ndarray or pandas.DataFrame
687
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
688
+ :type y_pred: numpy.ndarray
689
+ :param kwargs: Predictions, the algorithms will be identified using the keyword
690
+ :type kwargs: numpy.ndarray
691
+ :param num_samples: Number of bootstrap samples, default=500.
692
+ :type num_samples: int
693
+ :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
694
+ :type n_jobs: int
695
+ :param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
696
+ :type use_tqdm: bool
697
+ """
698
+
699
+ @wraps(stats.pearsonr)
700
+ def inner(y, hy):
701
+ return stats.pearsonr(y, hy,
702
+ alternative=alternative,
703
+ method=method).statistic
704
+
705
+ return Perf(y_true, *y_pred, score_func=inner, error_func=None,
706
+ num_samples=num_samples, n_jobs=n_jobs,
707
+ use_tqdm=use_tqdm,
708
+ **kwargs)
@@ -39,6 +39,21 @@ def test_f1_score():
39
39
  assert str(perf) is not None
40
40
 
41
41
 
42
+ def test_macro_f1_score():
43
+ """Test f1_score"""
44
+ from CompStats.metrics import macro_f1
45
+
46
+ X, y = load_iris(return_X_y=True)
47
+ _ = train_test_split(X, y, test_size=0.3)
48
+ X_train, X_val, y_train, y_val = _
49
+ ens = RandomForestClassifier().fit(X_train, y_train)
50
+ hy = ens.predict(X_val)
51
+ perf = macro_f1(y_val, forest=hy, num_samples=50)
52
+ assert isinstance(perf.statistic, float)
53
+ _ = metrics.f1_score(y_val, hy, average='macro')
54
+ assert _ == perf.statistic
55
+
56
+
42
57
  def test_accuracy_score():
43
58
  """Test f1_score"""
44
59
  from CompStats.metrics import accuracy_score
@@ -152,6 +167,23 @@ def test_precision_score():
152
167
  assert _ == perf.statistic
153
168
 
154
169
 
170
+ def test_macro_precision():
171
+ """Test macro_precision"""
172
+ from CompStats.metrics import macro_precision
173
+ import numpy as np
174
+
175
+ X, y = load_iris(return_X_y=True)
176
+ _ = train_test_split(X, y, test_size=0.3, stratify=y)
177
+ X_train, X_val, y_train, y_val = _
178
+ ens = RandomForestClassifier().fit(X_train, y_train)
179
+ hy = ens.predict(X_val)
180
+ perf = macro_precision(y_val,
181
+ forest=hy,
182
+ num_samples=50)
183
+ _ = metrics.precision_score(y_val, hy, average='macro')
184
+ assert _ == perf.statistic
185
+
186
+
155
187
  def test_recall_score():
156
188
  """Test recall_score"""
157
189
  from CompStats.metrics import recall_score
@@ -169,6 +201,23 @@ def test_recall_score():
169
201
  assert _ == perf.statistic
170
202
 
171
203
 
204
+ def test_macro_recall():
205
+ """Test macro_recall"""
206
+ from CompStats.metrics import macro_recall
207
+ import numpy as np
208
+
209
+ X, y = load_iris(return_X_y=True)
210
+ _ = train_test_split(X, y, test_size=0.3, stratify=y)
211
+ X_train, X_val, y_train, y_val = _
212
+ ens = RandomForestClassifier().fit(X_train, y_train)
213
+ hy = ens.predict(X_val)
214
+ perf = macro_recall(y_val,
215
+ forest=hy,
216
+ num_samples=50)
217
+ _ = metrics.recall_score(y_val, hy, average='macro')
218
+ assert _ == perf.statistic
219
+
220
+
172
221
  def test_jaccard_score():
173
222
  """jaccard_score"""
174
223
  from CompStats.metrics import jaccard_score
@@ -427,4 +476,21 @@ def test_d2_absolute_error_score():
427
476
  forest=hy,
428
477
  num_samples=50)
429
478
  _ = metrics.d2_absolute_error_score(y_val, hy)
430
- assert _ == perf.statistic
479
+ assert _ == perf.statistic
480
+
481
+
482
+ def test_pearsonr():
483
+ """test pearsonr"""
484
+ from CompStats.metrics import pearsonr
485
+ from scipy import stats
486
+
487
+ X, y = load_diabetes(return_X_y=True)
488
+ _ = train_test_split(X, y, test_size=0.3)
489
+ X_train, X_val, y_train, y_val = _
490
+ ens = RandomForestRegressor().fit(X_train, y_train)
491
+ hy = ens.predict(X_val)
492
+ perf = pearsonr(y_val,
493
+ forest=hy,
494
+ num_samples=50)
495
+ _ = stats.pearsonr(y_val, hy)
496
+ assert _.statistic == perf.statistic
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: CompStats
3
- Version: 0.1.12
3
+ Version: 0.1.13
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: CompStats
3
- Version: 0.1.12
3
+ Version: 0.1.13
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
5
  Project-URL: Homepage, https://compstats.readthedocs.io
6
6
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes