CompStats 0.1.12__tar.gz → 0.1.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/__init__.py +1 -1
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/metrics.py +110 -1
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_metrics.py +67 -1
- {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/PKG-INFO +1 -1
- {compstats-0.1.12 → compstats-0.1.13}/PKG-INFO +1 -1
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/interface.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/measurements.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/performance.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_interface.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats/utils.py +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/LICENSE +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/MANIFEST.in +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/README.rst +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/pyproject.toml +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/setup.cfg +0 -0
- {compstats-0.1.12 → compstats-0.1.13}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.13'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
from functools import wraps
|
|
15
15
|
from sklearn import metrics
|
|
16
|
+
from scipy import stats
|
|
16
17
|
from CompStats.interface import Perf
|
|
17
18
|
from CompStats.utils import metrics_docs
|
|
18
19
|
|
|
@@ -302,6 +303,81 @@ def d2_log_loss_score(y_true, *y_proba,
|
|
|
302
303
|
**kwargs)
|
|
303
304
|
|
|
304
305
|
|
|
306
|
+
def macro_f1(y_true, *y_pred, labels=None,
|
|
307
|
+
sample_weight=None, zero_division='warn',
|
|
308
|
+
num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
|
|
309
|
+
**kwargs):
|
|
310
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_f1`
|
|
311
|
+
|
|
312
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
313
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
314
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
|
|
315
|
+
:type y_pred: numpy.ndarray
|
|
316
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
317
|
+
:type kwargs: numpy.ndarray
|
|
318
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
319
|
+
:type num_samples: int
|
|
320
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
321
|
+
:type n_jobs: int
|
|
322
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
323
|
+
:type use_tqdm: bool
|
|
324
|
+
"""
|
|
325
|
+
return f1_score(y_true, *y_pred, labels=labels, average='macro',
|
|
326
|
+
sample_weight=sample_weight, zero_division=zero_division,
|
|
327
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
328
|
+
use_tqdm=use_tqdm, **kwargs)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def macro_recall(y_true, *y_pred, labels=None,
|
|
332
|
+
sample_weight=None, zero_division='warn',
|
|
333
|
+
num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
|
|
334
|
+
**kwargs):
|
|
335
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_recall`
|
|
336
|
+
|
|
337
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
338
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
339
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
|
|
340
|
+
:type y_pred: numpy.ndarray
|
|
341
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
342
|
+
:type kwargs: numpy.ndarray
|
|
343
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
344
|
+
:type num_samples: int
|
|
345
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
346
|
+
:type n_jobs: int
|
|
347
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
348
|
+
:type use_tqdm: bool
|
|
349
|
+
"""
|
|
350
|
+
return recall_score(y_true, *y_pred, labels=labels, average='macro',
|
|
351
|
+
sample_weight=sample_weight, zero_division=zero_division,
|
|
352
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
353
|
+
use_tqdm=use_tqdm, **kwargs)
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def macro_precision(y_true, *y_pred, labels=None,
|
|
357
|
+
sample_weight=None, zero_division='warn',
|
|
358
|
+
num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
|
|
359
|
+
**kwargs):
|
|
360
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_precision`
|
|
361
|
+
|
|
362
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
363
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
364
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
|
|
365
|
+
:type y_pred: numpy.ndarray
|
|
366
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
367
|
+
:type kwargs: numpy.ndarray
|
|
368
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
369
|
+
:type num_samples: int
|
|
370
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
371
|
+
:type n_jobs: int
|
|
372
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
373
|
+
:type use_tqdm: bool
|
|
374
|
+
"""
|
|
375
|
+
return precision_score(y_true, *y_pred, labels=labels, average='macro',
|
|
376
|
+
sample_weight=sample_weight, zero_division=zero_division,
|
|
377
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
378
|
+
use_tqdm=use_tqdm, **kwargs)
|
|
379
|
+
|
|
380
|
+
|
|
305
381
|
########################################################
|
|
306
382
|
#################### Regression ########################
|
|
307
383
|
########################################################
|
|
@@ -576,7 +652,6 @@ def mean_absolute_percentage_error(y_true,
|
|
|
576
652
|
**kwargs)
|
|
577
653
|
|
|
578
654
|
|
|
579
|
-
@metrics_docs(hy_name='y_pred', attr_name='score_func')
|
|
580
655
|
def d2_absolute_error_score(y_true,
|
|
581
656
|
*y_pred,
|
|
582
657
|
sample_weight=None,
|
|
@@ -597,3 +672,37 @@ def d2_absolute_error_score(y_true,
|
|
|
597
672
|
num_samples=num_samples, n_jobs=n_jobs,
|
|
598
673
|
use_tqdm=use_tqdm,
|
|
599
674
|
**kwargs)
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def pearsonr(y_true, *y_pred,
|
|
678
|
+
alternative='two-sided', method=None,
|
|
679
|
+
num_samples: int=500,
|
|
680
|
+
n_jobs: int=-1,
|
|
681
|
+
use_tqdm=True,
|
|
682
|
+
**kwargs):
|
|
683
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~scipy.stats.pearsonr` as :py:attr:`score_func.`
|
|
684
|
+
|
|
685
|
+
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
686
|
+
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
687
|
+
:param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`y_pred.`
|
|
688
|
+
:type y_pred: numpy.ndarray
|
|
689
|
+
:param kwargs: Predictions, the algorithms will be identified using the keyword
|
|
690
|
+
:type kwargs: numpy.ndarray
|
|
691
|
+
:param num_samples: Number of bootstrap samples, default=500.
|
|
692
|
+
:type num_samples: int
|
|
693
|
+
:param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
|
|
694
|
+
:type n_jobs: int
|
|
695
|
+
:param use_tqdm: Whether to use tqdm.tqdm to visualize the progress, default=True
|
|
696
|
+
:type use_tqdm: bool
|
|
697
|
+
"""
|
|
698
|
+
|
|
699
|
+
@wraps(stats.pearsonr)
|
|
700
|
+
def inner(y, hy):
|
|
701
|
+
return stats.pearsonr(y, hy,
|
|
702
|
+
alternative=alternative,
|
|
703
|
+
method=method).statistic
|
|
704
|
+
|
|
705
|
+
return Perf(y_true, *y_pred, score_func=inner, error_func=None,
|
|
706
|
+
num_samples=num_samples, n_jobs=n_jobs,
|
|
707
|
+
use_tqdm=use_tqdm,
|
|
708
|
+
**kwargs)
|
|
@@ -39,6 +39,21 @@ def test_f1_score():
|
|
|
39
39
|
assert str(perf) is not None
|
|
40
40
|
|
|
41
41
|
|
|
42
|
+
def test_macro_f1_score():
|
|
43
|
+
"""Test f1_score"""
|
|
44
|
+
from CompStats.metrics import macro_f1
|
|
45
|
+
|
|
46
|
+
X, y = load_iris(return_X_y=True)
|
|
47
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
48
|
+
X_train, X_val, y_train, y_val = _
|
|
49
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
50
|
+
hy = ens.predict(X_val)
|
|
51
|
+
perf = macro_f1(y_val, forest=hy, num_samples=50)
|
|
52
|
+
assert isinstance(perf.statistic, float)
|
|
53
|
+
_ = metrics.f1_score(y_val, hy, average='macro')
|
|
54
|
+
assert _ == perf.statistic
|
|
55
|
+
|
|
56
|
+
|
|
42
57
|
def test_accuracy_score():
|
|
43
58
|
"""Test f1_score"""
|
|
44
59
|
from CompStats.metrics import accuracy_score
|
|
@@ -152,6 +167,23 @@ def test_precision_score():
|
|
|
152
167
|
assert _ == perf.statistic
|
|
153
168
|
|
|
154
169
|
|
|
170
|
+
def test_macro_precision():
|
|
171
|
+
"""Test macro_precision"""
|
|
172
|
+
from CompStats.metrics import macro_precision
|
|
173
|
+
import numpy as np
|
|
174
|
+
|
|
175
|
+
X, y = load_iris(return_X_y=True)
|
|
176
|
+
_ = train_test_split(X, y, test_size=0.3, stratify=y)
|
|
177
|
+
X_train, X_val, y_train, y_val = _
|
|
178
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
179
|
+
hy = ens.predict(X_val)
|
|
180
|
+
perf = macro_precision(y_val,
|
|
181
|
+
forest=hy,
|
|
182
|
+
num_samples=50)
|
|
183
|
+
_ = metrics.precision_score(y_val, hy, average='macro')
|
|
184
|
+
assert _ == perf.statistic
|
|
185
|
+
|
|
186
|
+
|
|
155
187
|
def test_recall_score():
|
|
156
188
|
"""Test recall_score"""
|
|
157
189
|
from CompStats.metrics import recall_score
|
|
@@ -169,6 +201,23 @@ def test_recall_score():
|
|
|
169
201
|
assert _ == perf.statistic
|
|
170
202
|
|
|
171
203
|
|
|
204
|
+
def test_macro_recall():
|
|
205
|
+
"""Test macro_recall"""
|
|
206
|
+
from CompStats.metrics import macro_recall
|
|
207
|
+
import numpy as np
|
|
208
|
+
|
|
209
|
+
X, y = load_iris(return_X_y=True)
|
|
210
|
+
_ = train_test_split(X, y, test_size=0.3, stratify=y)
|
|
211
|
+
X_train, X_val, y_train, y_val = _
|
|
212
|
+
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
213
|
+
hy = ens.predict(X_val)
|
|
214
|
+
perf = macro_recall(y_val,
|
|
215
|
+
forest=hy,
|
|
216
|
+
num_samples=50)
|
|
217
|
+
_ = metrics.recall_score(y_val, hy, average='macro')
|
|
218
|
+
assert _ == perf.statistic
|
|
219
|
+
|
|
220
|
+
|
|
172
221
|
def test_jaccard_score():
|
|
173
222
|
"""jaccard_score"""
|
|
174
223
|
from CompStats.metrics import jaccard_score
|
|
@@ -427,4 +476,21 @@ def test_d2_absolute_error_score():
|
|
|
427
476
|
forest=hy,
|
|
428
477
|
num_samples=50)
|
|
429
478
|
_ = metrics.d2_absolute_error_score(y_val, hy)
|
|
430
|
-
assert _ == perf.statistic
|
|
479
|
+
assert _ == perf.statistic
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def test_pearsonr():
|
|
483
|
+
"""test pearsonr"""
|
|
484
|
+
from CompStats.metrics import pearsonr
|
|
485
|
+
from scipy import stats
|
|
486
|
+
|
|
487
|
+
X, y = load_diabetes(return_X_y=True)
|
|
488
|
+
_ = train_test_split(X, y, test_size=0.3)
|
|
489
|
+
X_train, X_val, y_train, y_val = _
|
|
490
|
+
ens = RandomForestRegressor().fit(X_train, y_train)
|
|
491
|
+
hy = ens.predict(X_val)
|
|
492
|
+
perf = pearsonr(y_val,
|
|
493
|
+
forest=hy,
|
|
494
|
+
num_samples=50)
|
|
495
|
+
_ = stats.pearsonr(y_val, hy)
|
|
496
|
+
assert _.statistic == perf.statistic
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.13
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.13
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|