CompStats 0.1.15__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/__init__.py +2 -2
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/bootstrap.py +42 -33
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/interface.py +299 -148
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/measurements.py +27 -8
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/metrics.py +664 -314
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/performance.py +114 -116
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/__init__.py +1 -1
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/test_bootstrap.py +5 -6
- compstats-0.2.0/CompStats/tests/test_docs.py +363 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/test_interface.py +233 -3
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/test_measurements.py +1 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/test_metrics.py +107 -16
- compstats-0.2.0/CompStats/tests/test_packaging.py +107 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/test_performance.py +4 -5
- compstats-0.2.0/CompStats/utils.py +132 -0
- compstats-0.2.0/CompStats.egg-info/PKG-INFO +156 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats.egg-info/SOURCES.txt +2 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats.egg-info/requires.txt +4 -0
- compstats-0.2.0/PKG-INFO +156 -0
- compstats-0.2.0/README.rst +125 -0
- {compstats-0.1.15 → compstats-0.2.0}/pyproject.toml +14 -2
- {compstats-0.1.15 → compstats-0.2.0}/setup.py +1 -1
- compstats-0.1.15/CompStats/utils.py +0 -85
- compstats-0.1.15/CompStats.egg-info/PKG-INFO +0 -129
- compstats-0.1.15/PKG-INFO +0 -129
- compstats-0.1.15/README.rst +0 -103
- {compstats-0.1.15 → compstats-0.2.0}/CompStats/tests/data.csv +0 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats.egg-info/dependency_links.txt +0 -0
- {compstats-0.1.15 → compstats-0.2.0}/CompStats.egg-info/top_level.txt +0 -0
- {compstats-0.1.15 → compstats-0.2.0}/LICENSE +0 -0
- {compstats-0.1.15 → compstats-0.2.0}/MANIFEST.in +0 -0
- {compstats-0.1.15 → compstats-0.2.0}/setup.cfg +0 -0
|
@@ -11,10 +11,10 @@
|
|
|
11
11
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
|
-
__version__ = '0.
|
|
14
|
+
__version__ = '0.2.0'
|
|
15
15
|
from CompStats.bootstrap import StatisticSamples
|
|
16
16
|
from CompStats.measurements import CI, SE, difference_p_value
|
|
17
17
|
from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
|
|
18
18
|
from CompStats.performance import performance_multiple_metrics, difference_multiple, plot_performance_multiple, plot_difference_multiple
|
|
19
19
|
from CompStats.performance import all_differences_multiple, plot_performance2, plot_difference2, plot_scatter_matrix
|
|
20
|
-
from CompStats.interface import Perf
|
|
20
|
+
from CompStats.interface import Perf
|
|
@@ -18,7 +18,7 @@ import numpy as np
|
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
class StatisticSamples:
|
|
21
|
-
"""Apply the statistic to `num_samples` samples taken with replacement
|
|
21
|
+
"""Apply the statistic to `num_samples` samples taken with replacement
|
|
22
22
|
from the population (arguments).
|
|
23
23
|
|
|
24
24
|
:param statistic: Statistic.
|
|
@@ -27,6 +27,8 @@ class StatisticSamples:
|
|
|
27
27
|
:type num_samples: int
|
|
28
28
|
:param n_jobs: Number of jobs to run in parallel, default=1.
|
|
29
29
|
:type n_jobs: int
|
|
30
|
+
:param BiB: Bigger is Better; a single bool for a scalar/vector statistic, or one bool per column when ``statistic`` returns the concatenation of several measures (see :py:class:`~CompStats.interface.Perf`'s multi-measure support).
|
|
31
|
+
:type BiB: bool or numpy.ndarray of bool
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
>>> from CompStats import StatisticSamples
|
|
@@ -44,14 +46,14 @@ class StatisticSamples:
|
|
|
44
46
|
"""
|
|
45
47
|
|
|
46
48
|
def __init__(self,
|
|
47
|
-
statistic: Callable[[np.ndarray], float]=np.mean,
|
|
48
|
-
num_samples: int=500,
|
|
49
|
-
n_jobs: int=1,
|
|
50
|
-
BiB: bool=True):
|
|
49
|
+
statistic: Callable[[np.ndarray], float] = np.mean,
|
|
50
|
+
num_samples: int = 500,
|
|
51
|
+
n_jobs: int = 1,
|
|
52
|
+
BiB: bool = True):
|
|
51
53
|
self.statistic = statistic
|
|
52
54
|
self.num_samples = num_samples
|
|
53
55
|
self.n_jobs = n_jobs
|
|
54
|
-
self.BiB = BiB #
|
|
56
|
+
self.BiB = BiB # Store the BiB parameter
|
|
55
57
|
self._samples = None
|
|
56
58
|
self._calls = {}
|
|
57
59
|
self._info = {}
|
|
@@ -60,7 +62,7 @@ class StatisticSamples:
|
|
|
60
62
|
def info(self):
|
|
61
63
|
"""Information about the samples"""
|
|
62
64
|
return self._info
|
|
63
|
-
|
|
65
|
+
|
|
64
66
|
@info.setter
|
|
65
67
|
def info(self, value):
|
|
66
68
|
self._info = value
|
|
@@ -70,7 +72,7 @@ class StatisticSamples:
|
|
|
70
72
|
return dict(statistic=self.statistic,
|
|
71
73
|
num_samples=self.num_samples,
|
|
72
74
|
n_jobs=self.n_jobs,
|
|
73
|
-
BiB=self.BiB) #
|
|
75
|
+
BiB=self.BiB) # Add BiB to the parameters
|
|
74
76
|
|
|
75
77
|
def __sklearn_clone__(self):
|
|
76
78
|
klass = self.__class__
|
|
@@ -83,7 +85,7 @@ class StatisticSamples:
|
|
|
83
85
|
def calls(self):
|
|
84
86
|
"""Dictionary containing the output of the calls when a name is given"""
|
|
85
87
|
return self._calls
|
|
86
|
-
|
|
88
|
+
|
|
87
89
|
@calls.setter
|
|
88
90
|
def calls(self, value):
|
|
89
91
|
self._calls = value
|
|
@@ -126,10 +128,15 @@ class StatisticSamples:
|
|
|
126
128
|
self._statistic_samples = value
|
|
127
129
|
|
|
128
130
|
def samples(self, N):
|
|
129
|
-
"""
|
|
130
|
-
|
|
131
|
+
"""Indices of the bootstrap samples for a population of size `N`. The
|
|
132
|
+
indices are drawn once per population size and cached, so every system
|
|
133
|
+
evaluated on a population of the same size uses the same resamples
|
|
134
|
+
(this is what makes the comparisons paired).
|
|
135
|
+
|
|
131
136
|
:param N: Population size.
|
|
132
137
|
:type N: int
|
|
138
|
+
:return: Array of shape (``num_samples``, `N`) with indices drawn with replacement.
|
|
139
|
+
:rtype: numpy.ndarray
|
|
133
140
|
"""
|
|
134
141
|
def inner(N):
|
|
135
142
|
_ = np.random.randint(N, size=(self.num_samples, N))
|
|
@@ -142,19 +149,23 @@ class StatisticSamples:
|
|
|
142
149
|
return inner(N)
|
|
143
150
|
except AttributeError:
|
|
144
151
|
return inner(N)
|
|
145
|
-
|
|
152
|
+
|
|
146
153
|
def keys(self):
|
|
147
|
-
"""
|
|
154
|
+
"""Names of the systems whose bootstrap samples are stored.
|
|
155
|
+
|
|
156
|
+
:return: Keys of ``calls``.
|
|
157
|
+
"""
|
|
148
158
|
return self.calls.keys()
|
|
149
159
|
|
|
150
160
|
def __getitem__(self, key):
|
|
161
|
+
"""Bootstrap samples of a system by name"""
|
|
151
162
|
return self.calls[key]
|
|
152
163
|
|
|
153
164
|
def __call__(self, *args: np.ndarray, name=None) -> np.ndarray:
|
|
154
|
-
"""Population where the bootstrap process will be performed.
|
|
165
|
+
"""Population where the bootstrap process will be performed.
|
|
155
166
|
|
|
156
167
|
:param *args: Population
|
|
157
|
-
:type
|
|
168
|
+
:type args: numpy.ndarray
|
|
158
169
|
"""
|
|
159
170
|
def inner(s):
|
|
160
171
|
_ = [arg[s] for arg in args]
|
|
@@ -176,16 +187,15 @@ class StatisticSamples:
|
|
|
176
187
|
value_name=value_name)
|
|
177
188
|
|
|
178
189
|
|
|
179
|
-
|
|
180
190
|
# class CI(StatisticSamples):
|
|
181
191
|
# """Compute the Confidence Interval of a statistic using bootstrap.
|
|
182
|
-
|
|
183
|
-
# :param alpha: :math:`[\\frac{\\alpha}{2}, 1 - \\frac{\\alpha}{2}]`.
|
|
192
|
+
|
|
193
|
+
# :param alpha: :math:`[\\frac{\\alpha}{2}, 1 - \\frac{\\alpha}{2}]`.
|
|
184
194
|
# :type alpha: float
|
|
185
195
|
|
|
186
196
|
# >>> from IngeoML import CI
|
|
187
197
|
# >>> from sklearn.metrics import accuracy_score
|
|
188
|
-
# >>> import numpy as np
|
|
198
|
+
# >>> import numpy as np
|
|
189
199
|
# >>> labels = np.r_[[0, 0, 0, 0, 0, 1, 1, 1, 1, 1]]
|
|
190
200
|
# >>> pred = np.r_[[0, 0, 1, 0, 0, 1, 1, 1, 0, 1]]
|
|
191
201
|
# >>> acc = CI(statistic=accuracy_score)
|
|
@@ -202,24 +212,24 @@ class StatisticSamples:
|
|
|
202
212
|
# """The interval is computed for :math:`[\\frac{\\alpha}{2}, 1 - \\frac{\\alpha}{2}]`.
|
|
203
213
|
# """
|
|
204
214
|
# return self._alpha
|
|
205
|
-
|
|
215
|
+
|
|
206
216
|
# @alpha.setter
|
|
207
217
|
# def alpha(self, value):
|
|
208
218
|
# self._alpha = value / 2
|
|
209
219
|
|
|
210
220
|
# def __call__(self, *args: np.ndarray) -> np.ndarray:
|
|
211
221
|
# B = super().__call__(*args)
|
|
212
|
-
# alpha = self.alpha
|
|
213
|
-
# return (np.percentile(B, alpha * 100, axis=0),
|
|
222
|
+
# alpha = self.alpha
|
|
223
|
+
# return (np.percentile(B, alpha * 100, axis=0),
|
|
214
224
|
# np.percentile(B, (1 - alpha) * 100, axis=0))
|
|
215
|
-
|
|
225
|
+
|
|
216
226
|
|
|
217
227
|
# class SE(StatisticSamples):
|
|
218
228
|
# """Compute the Standard Error of a statistic using bootstrap.
|
|
219
229
|
|
|
220
230
|
# >>> from IngeoML import SE
|
|
221
231
|
# >>> from sklearn.metrics import accuracy_score
|
|
222
|
-
# >>> import numpy as np
|
|
232
|
+
# >>> import numpy as np
|
|
223
233
|
# >>> labels = np.r_[[0, 0, 0, 0, 0, 1, 1, 1, 1, 1]]
|
|
224
234
|
# >>> pred = np.r_[[0, 0, 1, 0, 0, 1, 1, 1, 0, 1]]
|
|
225
235
|
# >>> se = SE(statistic=accuracy_score)
|
|
@@ -233,8 +243,8 @@ class StatisticSamples:
|
|
|
233
243
|
|
|
234
244
|
|
|
235
245
|
# class Difference(CI):
|
|
236
|
-
# def __init__(self, y: np.ndarray,
|
|
237
|
-
# algorithms: dict={},
|
|
246
|
+
# def __init__(self, y: np.ndarray,
|
|
247
|
+
# algorithms: dict={},
|
|
238
248
|
# performance: Callable[[np.ndarray, np.ndarray], float]=lambda y, hy: f1_score(y, hy, average='macro'),
|
|
239
249
|
# **kwargs) -> None:
|
|
240
250
|
# super(Difference, self).__init__(populations=algorithms, statistic=performance)
|
|
@@ -247,7 +257,7 @@ class StatisticSamples:
|
|
|
247
257
|
# @property
|
|
248
258
|
# def y(self):
|
|
249
259
|
# return self._y
|
|
250
|
-
|
|
260
|
+
|
|
251
261
|
# @y.setter
|
|
252
262
|
# def y(self, value):
|
|
253
263
|
# self._y = value
|
|
@@ -276,7 +286,7 @@ class StatisticSamples:
|
|
|
276
286
|
# delta = perf(y, algs[self.best]) - perf(y, algs[key])
|
|
277
287
|
# self._delta[key] = delta
|
|
278
288
|
# return delta
|
|
279
|
-
|
|
289
|
+
|
|
280
290
|
# def samples(self, key):
|
|
281
291
|
# if key in self.statistic_samples:
|
|
282
292
|
# return self.statistic_samples[key]
|
|
@@ -285,12 +295,12 @@ class StatisticSamples:
|
|
|
285
295
|
# output = np.array([self.statistic(y[s], data[s])
|
|
286
296
|
# for s in self.bootstrap])
|
|
287
297
|
# self.statistic_samples[key] = output
|
|
288
|
-
# return output
|
|
289
|
-
|
|
298
|
+
# return output
|
|
299
|
+
|
|
290
300
|
# @property
|
|
291
301
|
# def best_performance(self):
|
|
292
302
|
# return self.samples(self.best)
|
|
293
|
-
|
|
303
|
+
|
|
294
304
|
# def distribution(self, key):
|
|
295
305
|
# best = self.best
|
|
296
306
|
# assert key != best
|
|
@@ -320,11 +330,10 @@ class StatisticSamples:
|
|
|
320
330
|
# else:
|
|
321
331
|
# self._pvalue_l[key] = c
|
|
322
332
|
# return c
|
|
323
|
-
|
|
333
|
+
|
|
324
334
|
# def sort(self, side='right'):
|
|
325
335
|
# best = self.best
|
|
326
336
|
# algs = [(k, self.pvalue(k, side=side))
|
|
327
337
|
# for k in self.populations if k != best]
|
|
328
338
|
# algs.sort(key=lambda x: x[1], reverse=True)
|
|
329
339
|
# return [k for k, _ in algs]
|
|
330
|
-
|