CompStats 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CompStats/__init__.py +3 -2
- CompStats/bootstrap.py +28 -0
- CompStats/measurements.py +8 -1
- CompStats/performance.py +84 -5
- CompStats/tests/data.csv +1581 -0
- CompStats/tests/test_measurements.py +16 -2
- CompStats/tests/test_performance.py +20 -1
- {CompStats-0.0.2.dist-info → CompStats-0.0.4.dist-info}/METADATA +1 -1
- CompStats-0.0.4.dist-info/RECORD +14 -0
- CompStats-0.0.2.dist-info/RECORD +0 -13
- {CompStats-0.0.2.dist-info → CompStats-0.0.4.dist-info}/LICENSE +0 -0
- {CompStats-0.0.2.dist-info → CompStats-0.0.4.dist-info}/WHEEL +0 -0
- {CompStats-0.0.2.dist-info → CompStats-0.0.4.dist-info}/top_level.txt +0 -0
CompStats/__init__.py
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.0.
|
|
14
|
+
__version__ = '0.0.4'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
|
-
from CompStats.measurements import CI
|
|
16
|
+
from CompStats.measurements import CI, difference_p_value
|
|
17
|
+
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
CompStats/bootstrap.py
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
from typing import Callable
|
|
15
15
|
from joblib import delayed, Parallel
|
|
16
|
+
from copy import copy
|
|
16
17
|
import numpy as np
|
|
17
18
|
|
|
18
19
|
|
|
@@ -51,11 +52,38 @@ class StatisticSamples:
|
|
|
51
52
|
self.n_jobs = n_jobs
|
|
52
53
|
self._samples = None
|
|
53
54
|
self._calls = {}
|
|
55
|
+
self._info = {}
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def info(self):
|
|
59
|
+
"""Information about the samples"""
|
|
60
|
+
return self._info
|
|
61
|
+
|
|
62
|
+
@info.setter
|
|
63
|
+
def info(self, value):
|
|
64
|
+
self._info = value
|
|
65
|
+
|
|
66
|
+
def get_params(self):
|
|
67
|
+
"""Parameters"""
|
|
68
|
+
return dict(statistic=self.statistic,
|
|
69
|
+
num_samples=self.num_samples,
|
|
70
|
+
n_jobs=self.n_jobs)
|
|
71
|
+
|
|
72
|
+
def __sklearn_clone__(self):
|
|
73
|
+
klass = self.__class__
|
|
74
|
+
params = self.get_params()
|
|
75
|
+
ins = klass(**params)
|
|
76
|
+
ins.info = copy(self.info)
|
|
77
|
+
return ins
|
|
54
78
|
|
|
55
79
|
@property
|
|
56
80
|
def calls(self):
|
|
57
81
|
"""Dictionary containing the output of the calls when a name is given"""
|
|
58
82
|
return self._calls
|
|
83
|
+
|
|
84
|
+
@calls.setter
|
|
85
|
+
def calls(self, value):
|
|
86
|
+
self._calls = value
|
|
59
87
|
|
|
60
88
|
@property
|
|
61
89
|
def n_jobs(self):
|
CompStats/measurements.py
CHANGED
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
import numpy as np
|
|
15
|
-
|
|
15
|
+
import pandas as pd
|
|
16
|
+
from CompStats.bootstrap import StatisticSamples
|
|
16
17
|
|
|
17
18
|
def CI(samples: np.ndarray, alpha=0.05):
|
|
18
19
|
"""Compute the Confidence Interval of a statistic using bootstrap.
|
|
@@ -34,3 +35,9 @@ def CI(samples: np.ndarray, alpha=0.05):
|
|
|
34
35
|
alpha = alpha / 2
|
|
35
36
|
return (np.percentile(samples, alpha * 100, axis=0),
|
|
36
37
|
np.percentile(samples, (1 - alpha) * 100, axis=0))
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def difference_p_value(statistic_samples: StatisticSamples):
|
|
41
|
+
"""Compute the difference p-value"""
|
|
42
|
+
return {k: (v > 2 * np.mean(v)).mean()
|
|
43
|
+
for k, v in statistic_samples.calls.items()}
|
CompStats/performance.py
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
from sklearn.metrics import accuracy_score
|
|
15
|
+
from sklearn.base import clone
|
|
15
16
|
from typing import Callable
|
|
16
17
|
import pandas as pd
|
|
17
18
|
import numpy as np
|
|
@@ -24,6 +25,7 @@ def performance(data: pd.DataFrame,
|
|
|
24
25
|
gold: str='y',
|
|
25
26
|
score: Callable[[np.ndarray, np.ndarray], float]=accuracy_score,
|
|
26
27
|
statistic_samples: StatisticSamples=None) -> StatisticSamples:
|
|
28
|
+
"""Bootstrap samples of a performance score"""
|
|
27
29
|
if statistic_samples is None:
|
|
28
30
|
statistic_samples = StatisticSamples(statistic=score)
|
|
29
31
|
columns = data.columns
|
|
@@ -35,16 +37,93 @@ def performance(data: pd.DataFrame,
|
|
|
35
37
|
return statistic_samples
|
|
36
38
|
|
|
37
39
|
|
|
40
|
+
def difference(statistic_samples: StatisticSamples, best_index: int=-1):
|
|
41
|
+
"""Bootstrap samples of a difference in performnace"""
|
|
42
|
+
|
|
43
|
+
items = list(statistic_samples.calls.items())
|
|
44
|
+
perf = [(k, v, np.mean(v)) for k, v in items]
|
|
45
|
+
perf.sort(key=lambda x: x[-1])
|
|
46
|
+
best_name, best_perf, _ = perf[best_index]
|
|
47
|
+
diff = {}
|
|
48
|
+
for alg, alg_perf, _ in perf:
|
|
49
|
+
if alg == best_name:
|
|
50
|
+
continue
|
|
51
|
+
diff[alg] = best_perf - alg_perf
|
|
52
|
+
output = clone(statistic_samples)
|
|
53
|
+
output.calls = diff
|
|
54
|
+
output.info['best'] = best_name
|
|
55
|
+
return output
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def all_differences(statistic_samples: StatisticSamples, reverse: bool=True):
|
|
59
|
+
"""Calculates all possible differences in performance among algorithms and sorts by average performance"""
|
|
60
|
+
|
|
61
|
+
items = list(statistic_samples.calls.items())
|
|
62
|
+
# Calculamos el rendimiento medio y ordenamos los algoritmos basándonos en este
|
|
63
|
+
perf = [(k, v, np.mean(v)) for k, v in items]
|
|
64
|
+
perf.sort(key=lambda x: x[2], reverse=reverse) # Orden descendente por rendimiento medio
|
|
65
|
+
|
|
66
|
+
diffs = {} # Diccionario para guardar las diferencias
|
|
67
|
+
|
|
68
|
+
# Iteramos sobre todos los pares posibles de algoritmos ordenados
|
|
69
|
+
for i in range(len(perf)):
|
|
70
|
+
for j in range(i + 1, len(perf)):
|
|
71
|
+
name_i, perf_i, _ = perf[i]
|
|
72
|
+
name_j, perf_j, _ = perf[j]
|
|
73
|
+
|
|
74
|
+
# Diferencia de i a j
|
|
75
|
+
diff_key_i_to_j = f"{name_i} - {name_j}"
|
|
76
|
+
diffs[diff_key_i_to_j] = np.array(perf_i) - np.array(perf_j)
|
|
77
|
+
output = clone(statistic_samples)
|
|
78
|
+
output.calls = diffs
|
|
79
|
+
return output
|
|
80
|
+
|
|
81
|
+
|
|
38
82
|
def plot_performance(statistic_samples: StatisticSamples,
|
|
39
83
|
var_name='Algorithm', value_name='Score',
|
|
40
84
|
capsize=0.2, linestyle='none', kind='point',
|
|
41
|
-
sharex=False, **kwargs):
|
|
42
|
-
"""Plot the performance with the confidence intervals
|
|
85
|
+
sharex=False, CI=CI, **kwargs):
|
|
86
|
+
"""Plot the performance with the confidence intervals
|
|
87
|
+
|
|
88
|
+
>>> from CompStats import performance, plot_performance
|
|
89
|
+
>>> from CompStats.tests.test_performance import DATA
|
|
90
|
+
>>> from sklearn.metrics import f1_score
|
|
91
|
+
>>> import pandas as pd
|
|
92
|
+
>>> df = pd.read_csv(DATA)
|
|
93
|
+
>>> score = lambda y, hy: f1_score(y, hy, average='weighted')
|
|
94
|
+
>>> perf = performance(df, score=score)
|
|
95
|
+
>>> ins = plot_performance(perf)
|
|
96
|
+
"""
|
|
43
97
|
|
|
44
98
|
df2 = pd.DataFrame(statistic_samples.calls).melt(var_name=var_name,
|
|
45
99
|
value_name=value_name)
|
|
46
|
-
|
|
47
100
|
f_grid = sns.catplot(df2, x=value_name, y=var_name,
|
|
48
|
-
capsize=capsize, linestyle=linestyle,
|
|
101
|
+
capsize=capsize, linestyle=linestyle,
|
|
49
102
|
kind=kind, errorbar=CI, sharex=sharex, **kwargs)
|
|
50
|
-
return f_grid
|
|
103
|
+
return f_grid
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def plot_difference(statistic_samples: StatisticSamples,
|
|
107
|
+
var_name='Comparison', value_name='Difference',
|
|
108
|
+
set_refline=True, set_title=True,
|
|
109
|
+
**kwargs):
|
|
110
|
+
"""Plot the difference in performance with its confidence intervals
|
|
111
|
+
|
|
112
|
+
>>> from CompStats import performance, difference, plot_difference
|
|
113
|
+
>>> from CompStats.tests.test_performance import DATA
|
|
114
|
+
>>> from sklearn.metrics import f1_score
|
|
115
|
+
>>> import pandas as pd
|
|
116
|
+
>>> df = pd.read_csv(DATA)
|
|
117
|
+
>>> score = lambda y, hy: f1_score(y, hy, average='weighted')
|
|
118
|
+
>>> perf = performance(df, score=score)
|
|
119
|
+
>>> diff = difference(perf)
|
|
120
|
+
>>> ins = plot_difference(diff)
|
|
121
|
+
"""
|
|
122
|
+
f_grid = plot_performance(statistic_samples, var_name=var_name,
|
|
123
|
+
value_name=value_name, **kwargs)
|
|
124
|
+
if set_refline:
|
|
125
|
+
f_grid.refline(x=0)
|
|
126
|
+
if set_title:
|
|
127
|
+
best = statistic_samples.info['best']
|
|
128
|
+
f_grid.facet_axis(0, 0).set_title(f'Best: {best}')
|
|
129
|
+
return f_grid
|