CompStats 0.1.14__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {compstats-0.1.14 → compstats-0.2.0}/CompStats/__init__.py +2 -2
  2. {compstats-0.1.14 → compstats-0.2.0}/CompStats/bootstrap.py +42 -33
  3. {compstats-0.1.14 → compstats-0.2.0}/CompStats/interface.py +299 -148
  4. {compstats-0.1.14 → compstats-0.2.0}/CompStats/measurements.py +27 -8
  5. {compstats-0.1.14 → compstats-0.2.0}/CompStats/metrics.py +664 -314
  6. {compstats-0.1.14 → compstats-0.2.0}/CompStats/performance.py +114 -116
  7. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/__init__.py +1 -1
  8. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/test_bootstrap.py +5 -6
  9. compstats-0.2.0/CompStats/tests/test_docs.py +363 -0
  10. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/test_interface.py +233 -3
  11. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/test_measurements.py +1 -0
  12. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/test_metrics.py +107 -16
  13. compstats-0.2.0/CompStats/tests/test_packaging.py +107 -0
  14. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/test_performance.py +4 -5
  15. compstats-0.2.0/CompStats/utils.py +132 -0
  16. compstats-0.2.0/CompStats.egg-info/PKG-INFO +156 -0
  17. {compstats-0.1.14 → compstats-0.2.0}/CompStats.egg-info/SOURCES.txt +2 -0
  18. {compstats-0.1.14 → compstats-0.2.0}/CompStats.egg-info/requires.txt +4 -0
  19. compstats-0.2.0/PKG-INFO +156 -0
  20. compstats-0.2.0/README.rst +125 -0
  21. {compstats-0.1.14 → compstats-0.2.0}/pyproject.toml +15 -3
  22. {compstats-0.1.14 → compstats-0.2.0}/setup.py +1 -1
  23. compstats-0.1.14/CompStats/utils.py +0 -85
  24. compstats-0.1.14/CompStats.egg-info/PKG-INFO +0 -129
  25. compstats-0.1.14/PKG-INFO +0 -129
  26. compstats-0.1.14/README.rst +0 -103
  27. {compstats-0.1.14 → compstats-0.2.0}/CompStats/tests/data.csv +0 -0
  28. {compstats-0.1.14 → compstats-0.2.0}/CompStats.egg-info/dependency_links.txt +0 -0
  29. {compstats-0.1.14 → compstats-0.2.0}/CompStats.egg-info/top_level.txt +0 -0
  30. {compstats-0.1.14 → compstats-0.2.0}/LICENSE +0 -0
  31. {compstats-0.1.14 → compstats-0.2.0}/MANIFEST.in +0 -0
  32. {compstats-0.1.14 → compstats-0.2.0}/setup.cfg +0 -0
@@ -11,10 +11,10 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.14'
14
+ __version__ = '0.2.0'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
18
18
  from CompStats.performance import performance_multiple_metrics, difference_multiple, plot_performance_multiple, plot_difference_multiple
19
19
  from CompStats.performance import all_differences_multiple, plot_performance2, plot_difference2, plot_scatter_matrix
20
- from CompStats.interface import Perf
20
+ from CompStats.interface import Perf
@@ -18,7 +18,7 @@ import numpy as np
18
18
 
19
19
 
20
20
  class StatisticSamples:
21
- """Apply the statistic to `num_samples` samples taken with replacement
21
+ """Apply the statistic to `num_samples` samples taken with replacement
22
22
  from the population (arguments).
23
23
 
24
24
  :param statistic: Statistic.
@@ -27,6 +27,8 @@ class StatisticSamples:
27
27
  :type num_samples: int
28
28
  :param n_jobs: Number of jobs to run in parallel, default=1.
29
29
  :type n_jobs: int
30
+ :param BiB: Bigger is Better; a single bool for a scalar/vector statistic, or one bool per column when ``statistic`` returns the concatenation of several measures (see :py:class:`~CompStats.interface.Perf`'s multi-measure support).
31
+ :type BiB: bool or numpy.ndarray of bool
30
32
 
31
33
 
32
34
  >>> from CompStats import StatisticSamples
@@ -44,14 +46,14 @@ class StatisticSamples:
44
46
  """
45
47
 
46
48
  def __init__(self,
47
- statistic: Callable[[np.ndarray], float]=np.mean,
48
- num_samples: int=500,
49
- n_jobs: int=1,
50
- BiB: bool=True):
49
+ statistic: Callable[[np.ndarray], float] = np.mean,
50
+ num_samples: int = 500,
51
+ n_jobs: int = 1,
52
+ BiB: bool = True):
51
53
  self.statistic = statistic
52
54
  self.num_samples = num_samples
53
55
  self.n_jobs = n_jobs
54
- self.BiB = BiB # Guardar el parámetro BiB
56
+ self.BiB = BiB # Store the BiB parameter
55
57
  self._samples = None
56
58
  self._calls = {}
57
59
  self._info = {}
@@ -60,7 +62,7 @@ class StatisticSamples:
60
62
  def info(self):
61
63
  """Information about the samples"""
62
64
  return self._info
63
-
65
+
64
66
  @info.setter
65
67
  def info(self, value):
66
68
  self._info = value
@@ -70,7 +72,7 @@ class StatisticSamples:
70
72
  return dict(statistic=self.statistic,
71
73
  num_samples=self.num_samples,
72
74
  n_jobs=self.n_jobs,
73
- BiB=self.BiB) # Añadir BiB a los parámetros
75
+ BiB=self.BiB) # Add BiB to the parameters
74
76
 
75
77
  def __sklearn_clone__(self):
76
78
  klass = self.__class__
@@ -83,7 +85,7 @@ class StatisticSamples:
83
85
  def calls(self):
84
86
  """Dictionary containing the output of the calls when a name is given"""
85
87
  return self._calls
86
-
88
+
87
89
  @calls.setter
88
90
  def calls(self, value):
89
91
  self._calls = value
@@ -126,10 +128,15 @@ class StatisticSamples:
126
128
  self._statistic_samples = value
127
129
 
128
130
  def samples(self, N):
129
- """Samples.
130
-
131
+ """Indices of the bootstrap samples for a population of size `N`. The
132
+ indices are drawn once per population size and cached, so every system
133
+ evaluated on a population of the same size uses the same resamples
134
+ (this is what makes the comparisons paired).
135
+
131
136
  :param N: Population size.
132
137
  :type N: int
138
+ :return: Array of shape (``num_samples``, `N`) with indices drawn with replacement.
139
+ :rtype: numpy.ndarray
133
140
  """
134
141
  def inner(N):
135
142
  _ = np.random.randint(N, size=(self.num_samples, N))
@@ -142,19 +149,23 @@ class StatisticSamples:
142
149
  return inner(N)
143
150
  except AttributeError:
144
151
  return inner(N)
145
-
152
+
146
153
  def keys(self):
147
- """calls keys"""
154
+ """Names of the systems whose bootstrap samples are stored.
155
+
156
+ :return: Keys of ``calls``.
157
+ """
148
158
  return self.calls.keys()
149
159
 
150
160
  def __getitem__(self, key):
161
+ """Bootstrap samples of a system by name"""
151
162
  return self.calls[key]
152
163
 
153
164
  def __call__(self, *args: np.ndarray, name=None) -> np.ndarray:
154
- """Population where the bootstrap process will be performed.
165
+ """Population where the bootstrap process will be performed.
155
166
 
156
167
  :param *args: Population
157
- :type *args: np.ndarray
168
+ :type args: numpy.ndarray
158
169
  """
159
170
  def inner(s):
160
171
  _ = [arg[s] for arg in args]
@@ -176,16 +187,15 @@ class StatisticSamples:
176
187
  value_name=value_name)
177
188
 
178
189
 
179
-
180
190
  # class CI(StatisticSamples):
181
191
  # """Compute the Confidence Interval of a statistic using bootstrap.
182
-
183
- # :param alpha: :math:`[\\frac{\\alpha}{2}, 1 - \\frac{\\alpha}{2}]`.
192
+
193
+ # :param alpha: :math:`[\\frac{\\alpha}{2}, 1 - \\frac{\\alpha}{2}]`.
184
194
  # :type alpha: float
185
195
 
186
196
  # >>> from IngeoML import CI
187
197
  # >>> from sklearn.metrics import accuracy_score
188
- # >>> import numpy as np
198
+ # >>> import numpy as np
189
199
  # >>> labels = np.r_[[0, 0, 0, 0, 0, 1, 1, 1, 1, 1]]
190
200
  # >>> pred = np.r_[[0, 0, 1, 0, 0, 1, 1, 1, 0, 1]]
191
201
  # >>> acc = CI(statistic=accuracy_score)
@@ -202,24 +212,24 @@ class StatisticSamples:
202
212
  # """The interval is computed for :math:`[\\frac{\\alpha}{2}, 1 - \\frac{\\alpha}{2}]`.
203
213
  # """
204
214
  # return self._alpha
205
-
215
+
206
216
  # @alpha.setter
207
217
  # def alpha(self, value):
208
218
  # self._alpha = value / 2
209
219
 
210
220
  # def __call__(self, *args: np.ndarray) -> np.ndarray:
211
221
  # B = super().__call__(*args)
212
- # alpha = self.alpha
213
- # return (np.percentile(B, alpha * 100, axis=0),
222
+ # alpha = self.alpha
223
+ # return (np.percentile(B, alpha * 100, axis=0),
214
224
  # np.percentile(B, (1 - alpha) * 100, axis=0))
215
-
225
+
216
226
 
217
227
  # class SE(StatisticSamples):
218
228
  # """Compute the Standard Error of a statistic using bootstrap.
219
229
 
220
230
  # >>> from IngeoML import SE
221
231
  # >>> from sklearn.metrics import accuracy_score
222
- # >>> import numpy as np
232
+ # >>> import numpy as np
223
233
  # >>> labels = np.r_[[0, 0, 0, 0, 0, 1, 1, 1, 1, 1]]
224
234
  # >>> pred = np.r_[[0, 0, 1, 0, 0, 1, 1, 1, 0, 1]]
225
235
  # >>> se = SE(statistic=accuracy_score)
@@ -233,8 +243,8 @@ class StatisticSamples:
233
243
 
234
244
 
235
245
  # class Difference(CI):
236
- # def __init__(self, y: np.ndarray,
237
- # algorithms: dict={},
246
+ # def __init__(self, y: np.ndarray,
247
+ # algorithms: dict={},
238
248
  # performance: Callable[[np.ndarray, np.ndarray], float]=lambda y, hy: f1_score(y, hy, average='macro'),
239
249
  # **kwargs) -> None:
240
250
  # super(Difference, self).__init__(populations=algorithms, statistic=performance)
@@ -247,7 +257,7 @@ class StatisticSamples:
247
257
  # @property
248
258
  # def y(self):
249
259
  # return self._y
250
-
260
+
251
261
  # @y.setter
252
262
  # def y(self, value):
253
263
  # self._y = value
@@ -276,7 +286,7 @@ class StatisticSamples:
276
286
  # delta = perf(y, algs[self.best]) - perf(y, algs[key])
277
287
  # self._delta[key] = delta
278
288
  # return delta
279
-
289
+
280
290
  # def samples(self, key):
281
291
  # if key in self.statistic_samples:
282
292
  # return self.statistic_samples[key]
@@ -285,12 +295,12 @@ class StatisticSamples:
285
295
  # output = np.array([self.statistic(y[s], data[s])
286
296
  # for s in self.bootstrap])
287
297
  # self.statistic_samples[key] = output
288
- # return output
289
-
298
+ # return output
299
+
290
300
  # @property
291
301
  # def best_performance(self):
292
302
  # return self.samples(self.best)
293
-
303
+
294
304
  # def distribution(self, key):
295
305
  # best = self.best
296
306
  # assert key != best
@@ -320,11 +330,10 @@ class StatisticSamples:
320
330
  # else:
321
331
  # self._pvalue_l[key] = c
322
332
  # return c
323
-
333
+
324
334
  # def sort(self, side='right'):
325
335
  # best = self.best
326
336
  # algs = [(k, self.pvalue(k, side=side))
327
337
  # for k in self.populations if k != best]
328
338
  # algs.sort(key=lambda x: x[1], reverse=True)
329
339
  # return [k for k, _ in algs]
330
-