CompStats 0.1.8__tar.gz → 0.1.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/__init__.py +1 -1
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/interface.py +17 -14
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/utils.py +8 -5
- {compstats-0.1.8 → compstats-0.1.10}/CompStats.egg-info/PKG-INFO +1 -1
- {compstats-0.1.8 → compstats-0.1.10}/PKG-INFO +1 -1
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/bootstrap.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/measurements.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/metrics.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/performance.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/__init__.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/test_bootstrap.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/test_interface.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/test_measurements.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/test_metrics.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats/tests/test_performance.py +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats.egg-info/SOURCES.txt +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats.egg-info/requires.txt +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/LICENSE +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/MANIFEST.in +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/README.rst +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/pyproject.toml +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/setup.cfg +0 -0
- {compstats-0.1.8 → compstats-0.1.10}/setup.py +0 -0
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.1.
|
|
14
|
+
__version__ = '0.1.10'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
@@ -159,7 +159,7 @@ class Perf(object):
|
|
|
159
159
|
desc_se = [f'{k:0.4f}' for k in self.se]
|
|
160
160
|
desc_se = ', '.join(desc_se)
|
|
161
161
|
return f"<{self.__class__.__name__}({arg}={func_name}, statistic=[{desc}], se=[{desc_se}])>"
|
|
162
|
-
|
|
162
|
+
|
|
163
163
|
def __str__(self):
|
|
164
164
|
"""Prediction statistics with standard error in parenthesis"""
|
|
165
165
|
if not isinstance(self.statistic, dict):
|
|
@@ -358,10 +358,11 @@ class Perf(object):
|
|
|
358
358
|
>>> perf.plot()
|
|
359
359
|
"""
|
|
360
360
|
import seaborn as sns
|
|
361
|
-
if
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
361
|
+
if value_name is None:
|
|
362
|
+
if self.score_func is not None:
|
|
363
|
+
value_name = 'Score'
|
|
364
|
+
else:
|
|
365
|
+
value_name = 'Error'
|
|
365
366
|
df = self.dataframe(value_name=value_name, var_name=var_name,
|
|
366
367
|
alg_legend=alg_legend, perf_names=perf_names)
|
|
367
368
|
if var_name not in df.columns:
|
|
@@ -606,7 +607,8 @@ class Difference:
|
|
|
606
607
|
alg_legend:str='Algorithm',
|
|
607
608
|
sig_legend:str='Significant',
|
|
608
609
|
perf_names:str=None,
|
|
609
|
-
|
|
610
|
+
right:bool=True,
|
|
611
|
+
alpha:float=0.05):
|
|
610
612
|
"""Dataframe"""
|
|
611
613
|
if perf_names is None and isinstance(self.best, np.ndarray):
|
|
612
614
|
perf_names = [f'{alg}({k})'
|
|
@@ -617,18 +619,18 @@ class Difference:
|
|
|
617
619
|
perf_names=perf_names)
|
|
618
620
|
df[sig_legend] = False
|
|
619
621
|
if isinstance(self.best, str):
|
|
620
|
-
for name, p in self.p_value().items():
|
|
621
|
-
if p >=
|
|
622
|
+
for name, p in self.p_value(right=right).items():
|
|
623
|
+
if p >= alpha:
|
|
622
624
|
continue
|
|
623
625
|
df.loc[df[alg_legend] == name, sig_legend] = True
|
|
624
626
|
else:
|
|
625
|
-
p_values = self.p_value()
|
|
627
|
+
p_values = self.p_value(right=right)
|
|
626
628
|
systems = list(p_values.keys())
|
|
627
629
|
p_values = np.array([p_values[k] for k in systems])
|
|
628
630
|
for name, p_value in zip(perf_names, p_values.T):
|
|
629
631
|
mask = df[var_name] == name
|
|
630
632
|
for alg, p in zip(systems, p_value):
|
|
631
|
-
if p >=
|
|
633
|
+
if p >= alpha:
|
|
632
634
|
continue
|
|
633
635
|
_ = mask & (df[alg_legend] == alg)
|
|
634
636
|
df.loc[_, sig_legend] = True
|
|
@@ -639,7 +641,8 @@ class Difference:
|
|
|
639
641
|
alg_legend:str='Algorithm',
|
|
640
642
|
sig_legend:str='Significant',
|
|
641
643
|
perf_names:list=None,
|
|
642
|
-
|
|
644
|
+
alpha:float=0.05,
|
|
645
|
+
right:bool=True,
|
|
643
646
|
kind:str='point', linestyle:str='none',
|
|
644
647
|
col_wrap:int=3, capsize:float=0.2,
|
|
645
648
|
set_refline:bool=True,
|
|
@@ -668,12 +671,12 @@ class Difference:
|
|
|
668
671
|
alg_legend=alg_legend,
|
|
669
672
|
sig_legend=sig_legend,
|
|
670
673
|
perf_names=perf_names,
|
|
671
|
-
|
|
672
|
-
title = var_name
|
|
674
|
+
alpha=alpha, right=right)
|
|
675
|
+
title = var_name
|
|
673
676
|
if var_name not in df.columns:
|
|
674
677
|
var_name = None
|
|
675
678
|
col_wrap = None
|
|
676
|
-
ci = lambda x: measurements.CI(x, alpha=
|
|
679
|
+
ci = lambda x: measurements.CI(x, alpha=2*alpha)
|
|
677
680
|
f_grid = sns.catplot(df, x=value_name, errorbar=ci,
|
|
678
681
|
y=alg_legend, col=var_name,
|
|
679
682
|
kind=kind, linestyle=linestyle,
|
|
@@ -63,15 +63,18 @@ def dataframe(instance, value_name:str='Score',
|
|
|
63
63
|
perf_names:list=None):
|
|
64
64
|
"""Dataframe"""
|
|
65
65
|
import pandas as pd
|
|
66
|
+
statistic = instance.statistic
|
|
67
|
+
if not isinstance(statistic, dict):
|
|
68
|
+
iter = instance.statistic_samples.keys()
|
|
69
|
+
else:
|
|
70
|
+
iter = statistic
|
|
66
71
|
if isinstance(instance.best, str):
|
|
67
|
-
|
|
72
|
+
calls = instance.statistic_samples.calls
|
|
73
|
+
df = pd.DataFrame({k: calls[k]
|
|
74
|
+
for k in iter if k in calls})
|
|
68
75
|
return df.melt(var_name=alg_legend,
|
|
69
76
|
value_name=value_name)
|
|
70
77
|
df = pd.DataFrame()
|
|
71
|
-
if not isinstance(instance.statistic, dict):
|
|
72
|
-
iter = instance.statistic_samples.keys()
|
|
73
|
-
else:
|
|
74
|
-
iter = instance.statistic
|
|
75
78
|
for key in iter:
|
|
76
79
|
data = instance.statistic_samples[key]
|
|
77
80
|
_df = pd.DataFrame(data,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.10
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.2
|
|
2
2
|
Name: CompStats
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.10
|
|
4
4
|
Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
|
|
5
5
|
Project-URL: Homepage, https://compstats.readthedocs.io
|
|
6
6
|
Project-URL: Repository, https://github.com/INGEOTEC/CompStats
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|