CompStats 0.0.2__tar.gz → 0.0.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,6 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.0.2'
14
+ __version__ = '0.0.4'
15
15
  from CompStats.bootstrap import StatisticSamples
16
- from CompStats.measurements import CI
16
+ from CompStats.measurements import CI, difference_p_value
17
+ from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -13,6 +13,7 @@
13
13
  # limitations under the License.
14
14
  from typing import Callable
15
15
  from joblib import delayed, Parallel
16
+ from copy import copy
16
17
  import numpy as np
17
18
 
18
19
 
@@ -51,11 +52,38 @@ class StatisticSamples:
51
52
  self.n_jobs = n_jobs
52
53
  self._samples = None
53
54
  self._calls = {}
55
+ self._info = {}
56
+
57
+ @property
58
+ def info(self):
59
+ """Information about the samples"""
60
+ return self._info
61
+
62
+ @info.setter
63
+ def info(self, value):
64
+ self._info = value
65
+
66
+ def get_params(self):
67
+ """Parameters"""
68
+ return dict(statistic=self.statistic,
69
+ num_samples=self.num_samples,
70
+ n_jobs=self.n_jobs)
71
+
72
+ def __sklearn_clone__(self):
73
+ klass = self.__class__
74
+ params = self.get_params()
75
+ ins = klass(**params)
76
+ ins.info = copy(self.info)
77
+ return ins
54
78
 
55
79
  @property
56
80
  def calls(self):
57
81
  """Dictionary containing the output of the calls when a name is given"""
58
82
  return self._calls
83
+
84
+ @calls.setter
85
+ def calls(self, value):
86
+ self._calls = value
59
87
 
60
88
  @property
61
89
  def n_jobs(self):
@@ -12,7 +12,8 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import numpy as np
15
-
15
+ import pandas as pd
16
+ from CompStats.bootstrap import StatisticSamples
16
17
 
17
18
  def CI(samples: np.ndarray, alpha=0.05):
18
19
  """Compute the Confidence Interval of a statistic using bootstrap.
@@ -34,3 +35,9 @@ def CI(samples: np.ndarray, alpha=0.05):
34
35
  alpha = alpha / 2
35
36
  return (np.percentile(samples, alpha * 100, axis=0),
36
37
  np.percentile(samples, (1 - alpha) * 100, axis=0))
38
+
39
+
40
+ def difference_p_value(statistic_samples: StatisticSamples):
41
+ """Compute the difference p-value"""
42
+ return {k: (v > 2 * np.mean(v)).mean()
43
+ for k, v in statistic_samples.calls.items()}
@@ -0,0 +1,129 @@
1
+ # Copyright 2024 Sergio Nava Muñoz and Mario Graff Guerrero
2
+
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ from sklearn.metrics import accuracy_score
15
+ from sklearn.base import clone
16
+ from typing import Callable
17
+ import pandas as pd
18
+ import numpy as np
19
+ import seaborn as sns
20
+ from CompStats.bootstrap import StatisticSamples
21
+ from CompStats.measurements import CI
22
+
23
+
24
+ def performance(data: pd.DataFrame,
25
+ gold: str='y',
26
+ score: Callable[[np.ndarray, np.ndarray], float]=accuracy_score,
27
+ statistic_samples: StatisticSamples=None) -> StatisticSamples:
28
+ """Bootstrap samples of a performance score"""
29
+ if statistic_samples is None:
30
+ statistic_samples = StatisticSamples(statistic=score)
31
+ columns = data.columns
32
+ y = data[gold]
33
+ for column in columns:
34
+ if column == gold:
35
+ continue
36
+ statistic_samples(y, data[column], name=column)
37
+ return statistic_samples
38
+
39
+
40
+ def difference(statistic_samples: StatisticSamples, best_index: int=-1):
41
+ """Bootstrap samples of a difference in performnace"""
42
+
43
+ items = list(statistic_samples.calls.items())
44
+ perf = [(k, v, np.mean(v)) for k, v in items]
45
+ perf.sort(key=lambda x: x[-1])
46
+ best_name, best_perf, _ = perf[best_index]
47
+ diff = {}
48
+ for alg, alg_perf, _ in perf:
49
+ if alg == best_name:
50
+ continue
51
+ diff[alg] = best_perf - alg_perf
52
+ output = clone(statistic_samples)
53
+ output.calls = diff
54
+ output.info['best'] = best_name
55
+ return output
56
+
57
+
58
+ def all_differences(statistic_samples: StatisticSamples, reverse: bool=True):
59
+ """Calculates all possible differences in performance among algorithms and sorts by average performance"""
60
+
61
+ items = list(statistic_samples.calls.items())
62
+ # Calculamos el rendimiento medio y ordenamos los algoritmos basándonos en este
63
+ perf = [(k, v, np.mean(v)) for k, v in items]
64
+ perf.sort(key=lambda x: x[2], reverse=reverse) # Orden descendente por rendimiento medio
65
+
66
+ diffs = {} # Diccionario para guardar las diferencias
67
+
68
+ # Iteramos sobre todos los pares posibles de algoritmos ordenados
69
+ for i in range(len(perf)):
70
+ for j in range(i + 1, len(perf)):
71
+ name_i, perf_i, _ = perf[i]
72
+ name_j, perf_j, _ = perf[j]
73
+
74
+ # Diferencia de i a j
75
+ diff_key_i_to_j = f"{name_i} - {name_j}"
76
+ diffs[diff_key_i_to_j] = np.array(perf_i) - np.array(perf_j)
77
+ output = clone(statistic_samples)
78
+ output.calls = diffs
79
+ return output
80
+
81
+
82
+ def plot_performance(statistic_samples: StatisticSamples,
83
+ var_name='Algorithm', value_name='Score',
84
+ capsize=0.2, linestyle='none', kind='point',
85
+ sharex=False, CI=CI, **kwargs):
86
+ """Plot the performance with the confidence intervals
87
+
88
+ >>> from CompStats import performance, plot_performance
89
+ >>> from CompStats.tests.test_performance import DATA
90
+ >>> from sklearn.metrics import f1_score
91
+ >>> import pandas as pd
92
+ >>> df = pd.read_csv(DATA)
93
+ >>> score = lambda y, hy: f1_score(y, hy, average='weighted')
94
+ >>> perf = performance(df, score=score)
95
+ >>> ins = plot_performance(perf)
96
+ """
97
+
98
+ df2 = pd.DataFrame(statistic_samples.calls).melt(var_name=var_name,
99
+ value_name=value_name)
100
+ f_grid = sns.catplot(df2, x=value_name, y=var_name,
101
+ capsize=capsize, linestyle=linestyle,
102
+ kind=kind, errorbar=CI, sharex=sharex, **kwargs)
103
+ return f_grid
104
+
105
+
106
+ def plot_difference(statistic_samples: StatisticSamples,
107
+ var_name='Comparison', value_name='Difference',
108
+ set_refline=True, set_title=True,
109
+ **kwargs):
110
+ """Plot the difference in performance with its confidence intervals
111
+
112
+ >>> from CompStats import performance, difference, plot_difference
113
+ >>> from CompStats.tests.test_performance import DATA
114
+ >>> from sklearn.metrics import f1_score
115
+ >>> import pandas as pd
116
+ >>> df = pd.read_csv(DATA)
117
+ >>> score = lambda y, hy: f1_score(y, hy, average='weighted')
118
+ >>> perf = performance(df, score=score)
119
+ >>> diff = difference(perf)
120
+ >>> ins = plot_difference(diff)
121
+ """
122
+ f_grid = plot_performance(statistic_samples, var_name=var_name,
123
+ value_name=value_name, **kwargs)
124
+ if set_refline:
125
+ f_grid.refline(x=0)
126
+ if set_title:
127
+ best = statistic_samples.info['best']
128
+ f_grid.facet_axis(0, 0).set_title(f'Best: {best}')
129
+ return f_grid