CompStats 0.1.13__tar.gz → 0.1.14__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/__init__.py +1 -1
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/interface.py +54 -7
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/metrics.py +3 -3
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_interface.py +6 -2
- {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/PKG-INFO +2 -2
- {compstats-0.1.13 → compstats-0.1.14}/PKG-INFO +2 -2
- {compstats-0.1.13 → compstats-0.1.14}/pyproject.toml +1 -1
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/measurements.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/performance.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_metrics.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats/utils.py +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/LICENSE +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/MANIFEST.in +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/README.rst +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/setup.cfg +0 -0
- {compstats-0.1.13 → compstats-0.1.14}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.14'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -19,7 +19,7 @@ import numpy as np
|
|
|
19
19
|
from CompStats.bootstrap import StatisticSamples
|
|
20
20
|
from CompStats.utils import progress_bar
|
|
21
21
|
from CompStats import measurements
|
|
22
|
-
from CompStats.measurements import SE
|
|
22
|
+
from CompStats.measurements import SE, CI
|
|
23
23
|
from CompStats.utils import dataframe
|
|
24
24
|
|
|
25
25
|
|
|
@@ -270,7 +270,7 @@ class Perf(object):
|
|
|
270
270
|
best = data.argmin()
|
|
271
271
|
self._best = keys[best]
|
|
272
272
|
return self._best
|
|
273
|
-
|
|
273
|
+
|
|
274
274
|
@best.setter
|
|
275
275
|
def best(self, value):
|
|
276
276
|
self._best = value
|
|
@@ -279,7 +279,7 @@ class Perf(object):
|
|
|
279
279
|
def sorting_func(self):
|
|
280
280
|
"""Rank systems when multiple performances are used"""
|
|
281
281
|
return self._sorting_func
|
|
282
|
-
|
|
282
|
+
|
|
283
283
|
@sorting_func.setter
|
|
284
284
|
def sorting_func(self, value):
|
|
285
285
|
self._sorting_func = value
|
|
@@ -315,7 +315,7 @@ class Perf(object):
|
|
|
315
315
|
else:
|
|
316
316
|
self._statistic = dict(data)
|
|
317
317
|
return self._statistic
|
|
318
|
-
|
|
318
|
+
|
|
319
319
|
@statistic.setter
|
|
320
320
|
def statistic(self, value):
|
|
321
321
|
"""statistic setter"""
|
|
@@ -346,6 +346,30 @@ class Perf(object):
|
|
|
346
346
|
return list(output.values())[0]
|
|
347
347
|
return output
|
|
348
348
|
|
|
349
|
+
@property
|
|
350
|
+
def ci(self):
|
|
351
|
+
"""Confidence interval
|
|
352
|
+
|
|
353
|
+
>>> from sklearn.svm import LinearSVC
|
|
354
|
+
>>> from sklearn.datasets import load_iris
|
|
355
|
+
>>> from sklearn.model_selection import train_test_split
|
|
356
|
+
>>> from CompStats.interface import Perf
|
|
357
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
358
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
359
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
360
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
361
|
+
>>> hy = m.predict(X_val)
|
|
362
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
363
|
+
>>> perf = Perf(y_val, hy, name='LinearSVC')
|
|
364
|
+
>>> perf.ci
|
|
365
|
+
(np.float64(0.9333333333333332), np.float64(1.0))
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
output = CI(self.statistic_samples)
|
|
369
|
+
if len(output) == 1:
|
|
370
|
+
return list(output.values())[0]
|
|
371
|
+
return output
|
|
372
|
+
|
|
349
373
|
def plot(self, value_name:str=None,
|
|
350
374
|
var_name:str='Performance',
|
|
351
375
|
alg_legend:str='Algorithm',
|
|
@@ -359,6 +383,7 @@ class Perf(object):
|
|
|
359
383
|
winner_legend:str='Best',
|
|
360
384
|
tie_legend:str='Equivalent',
|
|
361
385
|
loser_legend:str='Different',
|
|
386
|
+
palette:object=None,
|
|
362
387
|
**kwargs):
|
|
363
388
|
"""plot with seaborn
|
|
364
389
|
|
|
@@ -403,10 +428,17 @@ class Perf(object):
|
|
|
403
428
|
ci = lambda x: measurements.CI(x, alpha=CI)
|
|
404
429
|
if comparison:
|
|
405
430
|
kwargs.update(dict(hue=comp_legend))
|
|
431
|
+
if palette is None:
|
|
432
|
+
pal = sns.color_palette("Paired")
|
|
433
|
+
palette = {winner_legend:pal[1],
|
|
434
|
+
tie_legend:pal[3],
|
|
435
|
+
loser_legend: pal[5]}
|
|
406
436
|
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
407
437
|
y=alg_legend, col=var_name,
|
|
408
438
|
kind=kind, linestyle=linestyle,
|
|
409
|
-
col_wrap=col_wrap, capsize=capsize,
|
|
439
|
+
col_wrap=col_wrap, capsize=capsize,
|
|
440
|
+
palette=palette,
|
|
441
|
+
**kwargs)
|
|
410
442
|
return f_grid
|
|
411
443
|
|
|
412
444
|
def dataframe(self, comparison:bool=False,
|
|
@@ -420,7 +452,22 @@ class Perf(object):
|
|
|
420
452
|
tie_legend:str='Equivalent',
|
|
421
453
|
loser_legend:str='Different',
|
|
422
454
|
perf_names:str=None):
|
|
423
|
-
"""Dataframe
|
|
455
|
+
"""Dataframe
|
|
456
|
+
|
|
457
|
+
>>> from sklearn.svm import LinearSVC
|
|
458
|
+
>>> from sklearn.ensemble import RandomForestClassifier
|
|
459
|
+
>>> from sklearn.datasets import load_iris
|
|
460
|
+
>>> from sklearn.model_selection import train_test_split
|
|
461
|
+
>>> from CompStats.interface import Perf
|
|
462
|
+
>>> X, y = load_iris(return_X_y=True)
|
|
463
|
+
>>> _ = train_test_split(X, y, test_size=0.3)
|
|
464
|
+
>>> X_train, X_val, y_train, y_val = _
|
|
465
|
+
>>> m = LinearSVC().fit(X_train, y_train)
|
|
466
|
+
>>> hy = m.predict(X_val)
|
|
467
|
+
>>> ens = RandomForestClassifier().fit(X_train, y_train)
|
|
468
|
+
>>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
|
|
469
|
+
>>> df = perf.dataframe()
|
|
470
|
+
"""
|
|
424
471
|
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
425
472
|
func_name = self.statistic_func.__name__
|
|
426
473
|
perf_names = [f'{func_name}({i})'
|
|
@@ -624,7 +671,7 @@ class Difference:
|
|
|
624
671
|
return self.statistic[self.best]
|
|
625
672
|
keys = np.unique(self.best)
|
|
626
673
|
statistic = np.array([self.statistic[k]
|
|
627
|
-
|
|
674
|
+
for k in keys])
|
|
628
675
|
m = {v: k for k, v in enumerate(keys)}
|
|
629
676
|
best = np.array([m[x] for x in self.best])
|
|
630
677
|
return statistic[best, np.arange(best.shape[0])]
|
|
@@ -307,7 +307,7 @@ def macro_f1(y_true, *y_pred, labels=None,
|
|
|
307
307
|
sample_weight=None, zero_division='warn',
|
|
308
308
|
num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
|
|
309
309
|
**kwargs):
|
|
310
|
-
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.
|
|
310
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.f1_score`
|
|
311
311
|
|
|
312
312
|
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
313
313
|
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
@@ -332,7 +332,7 @@ def macro_recall(y_true, *y_pred, labels=None,
|
|
|
332
332
|
sample_weight=None, zero_division='warn',
|
|
333
333
|
num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
|
|
334
334
|
**kwargs):
|
|
335
|
-
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.
|
|
335
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.recall_score`
|
|
336
336
|
|
|
337
337
|
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
338
338
|
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
@@ -357,7 +357,7 @@ def macro_precision(y_true, *y_pred, labels=None,
|
|
|
357
357
|
sample_weight=None, zero_division='warn',
|
|
358
358
|
num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
|
|
359
359
|
**kwargs):
|
|
360
|
-
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.
|
|
360
|
+
""":py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.precision_score`
|
|
361
361
|
|
|
362
362
|
:param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
|
|
363
363
|
:type y_true: numpy.ndarray or pandas.DataFrame
|
|
@@ -81,7 +81,7 @@ def test_Perf_dataframe():
|
|
|
81
81
|
from CompStats.metrics import f1_score
|
|
82
82
|
|
|
83
83
|
X, y = load_digits(return_X_y=True)
|
|
84
|
-
_ = train_test_split(X, y, test_size=0.3)
|
|
84
|
+
_ = train_test_split(X, y, test_size=0.3, random_state=0)
|
|
85
85
|
X_train, X_val, y_train, y_val = _
|
|
86
86
|
ens = RandomForestClassifier().fit(X_train, y_train)
|
|
87
87
|
nb = GaussianNB().fit(X_train, y_train)
|
|
@@ -121,6 +121,7 @@ def test_Perf_plot_multi():
|
|
|
121
121
|
f_grid = score.plot()
|
|
122
122
|
assert f_grid is not None
|
|
123
123
|
|
|
124
|
+
|
|
124
125
|
def test_Perf_statistic_one():
|
|
125
126
|
"""Test Perf statistic one alg"""
|
|
126
127
|
from CompStats.metrics import f1_score
|
|
@@ -142,6 +143,9 @@ def test_Perf_statistic_one():
|
|
|
142
143
|
assert isinstance(score.statistic, float)
|
|
143
144
|
assert isinstance(str(score), str)
|
|
144
145
|
assert isinstance(score.se, float)
|
|
146
|
+
assert isinstance(score.ci, tuple)
|
|
147
|
+
assert len(score.ci) == 2
|
|
148
|
+
|
|
145
149
|
|
|
146
150
|
def test_Perf_best():
|
|
147
151
|
"""Test Perf best"""
|
|
@@ -191,7 +195,7 @@ def test_difference_best():
|
|
|
191
195
|
score(svm.predict(X_val), name='svm')
|
|
192
196
|
diff = score.difference()
|
|
193
197
|
assert isinstance(diff.best, str)
|
|
194
|
-
|
|
198
|
+
|
|
195
199
|
|
|
196
200
|
def test_difference_str__():
|
|
197
201
|
"""Test f1_score"""
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.14
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
|
+
License-Expression: Apache-2.0
|
|
5
6
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
7
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
7
8
|
Project-URL: Issues, https://github.com/INGEOTEC/CompStats/issues
|
|
@@ -10,7 +11,6 @@ Classifier: Environment :: Console
|
|
|
10
11
|
Classifier: Intended Audience :: Developers
|
|
11
12
|
Classifier: Intended Audience :: Information Technology
|
|
12
13
|
Classifier: Intended Audience :: Science/Research
|
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Programming Language :: Python
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.14
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
|
+
License-Expression: Apache-2.0
|
|
5
6
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
7
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
7
8
|
Project-URL: Issues, https://github.com/INGEOTEC/CompStats/issues
|
|
@@ -10,7 +11,6 @@ Classifier: Environment :: Console
|
|
|
10
11
|
Classifier: Intended Audience :: Developers
|
|
11
12
|
Classifier: Intended Audience :: Information Technology
|
|
12
13
|
Classifier: Intended Audience :: Science/Research
|
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
14
14
|
Classifier: Operating System :: OS Independent
|
|
15
15
|
Classifier: Programming Language :: Python
|
|
16
16
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
name = 'CompStats'
|
|
3
3
|
description = 'CompStats implements an evaluation methodology for statistically analyzing competition results and competition'
|
|
4
4
|
readme = "README.rst"
|
|
5
|
+
license = "Apache-2.0"
|
|
5
6
|
dependencies = [
|
|
6
7
|
'numpy',
|
|
7
8
|
'scikit-learn>=1.3.0',
|
|
@@ -17,7 +18,6 @@ classifiers = [
|
|
|
17
18
|
"Intended Audience :: Developers",
|
|
18
19
|
"Intended Audience :: Information Technology",
|
|
19
20
|
"Intended Audience :: Science/Research",
|
|
20
|
-
"License :: OSI Approved :: MIT License",
|
|
21
21
|
"Operating System :: OS Independent",
|
|
22
22
|
"Programming Language :: Python",
|
|
23
23
|
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|