CompStats 0.1.3__tar.gz → 0.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. {compstats-0.1.3 → compstats-0.1.4}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.3 → compstats-0.1.4}/CompStats/interface.py +33 -26
  3. compstats-0.1.4/CompStats/metrics.py +553 -0
  4. {compstats-0.1.3 → compstats-0.1.4}/CompStats/performance.py +29 -20
  5. {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_interface.py +2 -2
  6. compstats-0.1.4/CompStats/tests/test_metrics.py +453 -0
  7. compstats-0.1.4/CompStats/utils.py +55 -0
  8. {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/PKG-INFO +67 -2
  9. {compstats-0.1.3 → compstats-0.1.4}/PKG-INFO +67 -2
  10. compstats-0.1.4/README.rst +93 -0
  11. compstats-0.1.3/CompStats/metrics.py +0 -197
  12. compstats-0.1.3/CompStats/tests/test_metrics.py +0 -56
  13. compstats-0.1.3/CompStats/utils.py +0 -25
  14. compstats-0.1.3/README.rst +0 -28
  15. {compstats-0.1.3 → compstats-0.1.4}/CompStats/bootstrap.py +0 -0
  16. {compstats-0.1.3 → compstats-0.1.4}/CompStats/measurements.py +0 -0
  17. {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/__init__.py +0 -0
  18. {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/data.csv +0 -0
  19. {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_bootstrap.py +0 -0
  20. {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_measurements.py +0 -0
  21. {compstats-0.1.3 → compstats-0.1.4}/CompStats/tests/test_performance.py +0 -0
  22. {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/SOURCES.txt +0 -0
  23. {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/dependency_links.txt +0 -0
  24. {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/requires.txt +0 -0
  25. {compstats-0.1.3 → compstats-0.1.4}/CompStats.egg-info/top_level.txt +0 -0
  26. {compstats-0.1.3 → compstats-0.1.4}/LICENSE +0 -0
  27. {compstats-0.1.3 → compstats-0.1.4}/MANIFEST.in +0 -0
  28. {compstats-0.1.3 → compstats-0.1.4}/pyproject.toml +0 -0
  29. {compstats-0.1.3 → compstats-0.1.4}/setup.cfg +0 -0
  30. {compstats-0.1.3 → compstats-0.1.4}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.3'
14
+ __version__ = '0.1.4'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -31,8 +31,8 @@ class Perf(object):
31
31
  :type score_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
32
32
  :param error_func: Function to measure the performance where the best algorithm has the lowest value.
33
33
  :type error_func: Function where the first argument is :math:`y` and the second is :math:`\\hat{y}.`
34
- :param args: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
35
- :type args: numpy.ndarray
34
+ :param y_pred: Predictions, the algorithms will be identified with alg-k where k=1 is the first argument included in :py:attr:`args.`
35
+ :type y_pred: numpy.ndarray
36
36
  :param kwargs: Predictions, the algorithms will be identified using the keyword
37
37
  :type kwargs: numpy.ndarray
38
38
  :param n_jobs: Number of jobs to compute the statistic, default=-1 corresponding to use all threads.
@@ -59,19 +59,21 @@ class Perf(object):
59
59
  >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
60
60
  >>> perf
61
61
  <Perf>
62
- Prediction statistics with standard error
63
- alg-1 = 1.000 (0.000)
64
- forest = 0.946 (0.038)
62
+ Statistic with its standard error (se)
63
+ statistic (se)
64
+ 0.9792 (0.0221) <= alg-1
65
+ 0.9744 (0.0246) <= forest
65
66
 
66
67
  If an algorithm's prediction is missing, this can be included by calling the instance, as can be seen in the following instruction. Note that the algorithm's name can also be given with the keyword :py:attr:`name.`
67
68
 
68
69
  >>> lr = LogisticRegression().fit(X_train, y_train)
69
70
  >>> perf(lr.predict(X_val), name='Log. Reg.')
70
71
  <Perf>
71
- Prediction statistics with standard error
72
- alg-1 = 1.000 (0.000)
73
- forest = 0.946 (0.038)
74
- Log. Reg. = 0.946 (0.038)
72
+ Statistic with its standard error (se)
73
+ statistic (se)
74
+ 1.0000 (0.0000) <= Log. Reg.
75
+ 0.9792 (0.0221) <= alg-1
76
+ 0.9744 (0.0246) <= forest
75
77
 
76
78
  The performance function used to compare the algorithms can be changed, and the same bootstrap samples would be used if the instance were cloned. Consequently, the values are computed using the same samples, as can be seen in the following example.
77
79
 
@@ -79,13 +81,14 @@ class Perf(object):
79
81
  >>> perf_error.error_func = lambda y, hy: (y != hy).mean()
80
82
  >>> perf_error
81
83
  <Perf>
82
- Prediction statistics with standard error
83
- alg-1 = 0.000 (0.000)
84
- forest = 0.044 (0.030)
85
- Log. Reg. = 0.044 (0.030)
84
+ Statistic with its standard error (se)
85
+ statistic (se)
86
+ 0.0000 (0.0000) <= Log. Reg.
87
+ 0.0222 (0.0237) <= alg-1
88
+ 0.0222 (0.0215) <= forest
86
89
 
87
90
  """
88
- def __init__(self, y_true, *args,
91
+ def __init__(self, y_true, *y_pred,
89
92
  score_func=balanced_accuracy_score,
90
93
  error_func=None,
91
94
  num_samples: int=500,
@@ -96,7 +99,7 @@ class Perf(object):
96
99
  self.score_func = score_func
97
100
  self.error_func = error_func
98
101
  algs = {}
99
- for k, v in enumerate(args):
102
+ for k, v in enumerate(y_pred):
100
103
  algs[f'alg-{k+1}'] = np.asanyarray(v)
101
104
  algs.update(**kwargs)
102
105
  self.predictions = algs
@@ -137,7 +140,7 @@ class Perf(object):
137
140
  ins.predictions = dict(self.predictions)
138
141
  ins._statistic_samples._samples = self.statistic_samples._samples
139
142
  return ins
140
-
143
+
141
144
  def __repr__(self):
142
145
  """Prediction statistics with standard error in parenthesis"""
143
146
  return f"<{self.__class__.__name__}>\n{self}"
@@ -145,10 +148,11 @@ class Perf(object):
145
148
  def __str__(self):
146
149
  """Prediction statistics with standard error in parenthesis"""
147
150
 
148
- se = self.se()
149
- output = ["Prediction statistics with standard error"]
150
- for key, value in self.statistic().items():
151
- output.append(f'{key} = {value:0.3f} ({se[key]:0.3f})')
151
+ se = self.se
152
+ output = ["Statistic with its standard error (se)"]
153
+ output.append("statistic (se)")
154
+ for key, value in self.statistic.items():
155
+ output.append(f'{value:0.4f} ({se[key]:0.4f}) <= {key}')
152
156
  return "\n".join(output)
153
157
 
154
158
  def __call__(self, y_pred, name=None):
@@ -196,7 +200,7 @@ class Perf(object):
196
200
  continue
197
201
  diff[k] = sign * (base - v)
198
202
  diff_ins = Difference(statistic_samples=clone(self.statistic_samples),
199
- statistic=self.statistic(),
203
+ statistic=self.statistic,
200
204
  best=self.best[0])
201
205
  diff_ins.statistic_samples.calls = diff
202
206
  diff_ins.statistic_samples.info['best'] = self.best[0]
@@ -209,12 +213,13 @@ class Perf(object):
209
213
  try:
210
214
  return self._best
211
215
  except AttributeError:
212
- statistic = [(k, v) for k, v in self.statistic().items()]
216
+ statistic = [(k, v) for k, v in self.statistic.items()]
213
217
  statistic = sorted(statistic, key=lambda x: x[1],
214
218
  reverse=self.statistic_samples.BiB)
215
219
  self._best = statistic[0]
216
220
  return self._best
217
221
 
222
+ @property
218
223
  def statistic(self):
219
224
  """Statistic
220
225
 
@@ -230,7 +235,7 @@ class Perf(object):
230
235
  >>> hy = m.predict(X_val)
231
236
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
232
237
  >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
233
- >>> perf.statistic()
238
+ >>> perf.statistic
234
239
  {'alg-1': 1.0, 'forest': 0.9500891265597148}
235
240
  """
236
241
 
@@ -239,6 +244,7 @@ class Perf(object):
239
244
  key=lambda x: x[1], reverse=self.statistic_samples.BiB)
240
245
  return dict(data)
241
246
 
247
+ @property
242
248
  def se(self):
243
249
  """Standard Error
244
250
 
@@ -254,9 +260,10 @@ class Perf(object):
254
260
  >>> hy = m.predict(X_val)
255
261
  >>> ens = RandomForestClassifier().fit(X_train, y_train)
256
262
  >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
257
- >>> print(perf.se())
263
+ >>> perf.se
258
264
  {'alg-1': 0.0, 'forest': 0.026945730782184187}
259
265
  """
266
+
260
267
  return SE(self.statistic_samples)
261
268
 
262
269
  def plot(self, **kwargs):
@@ -405,7 +412,7 @@ class Difference:
405
412
  >>> diff
406
413
  <Difference>
407
414
  difference p-values w.r.t alg-1
408
- forest 0.3
415
+ 0.0780 <= forest
409
416
  """
410
417
 
411
418
  statistic_samples:StatisticSamples=None
@@ -420,7 +427,7 @@ class Difference:
420
427
  """p-value"""
421
428
  output = [f"difference p-values w.r.t {self.best}"]
422
429
  for k, v in self.p_value().items():
423
- output.append(f'{k} {v}')
430
+ output.append(f'{v:0.4f} <= {k}')
424
431
  return "\n".join(output)
425
432
 
426
433
  def p_value(self):