CompStats 0.1.13__tar.gz → 0.1.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {compstats-0.1.13 → compstats-0.1.14}/CompStats/__init__.py +1 -1
  2. {compstats-0.1.13 → compstats-0.1.14}/CompStats/interface.py +54 -7
  3. {compstats-0.1.13 → compstats-0.1.14}/CompStats/metrics.py +3 -3
  4. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_interface.py +6 -2
  5. {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/PKG-INFO +2 -2
  6. {compstats-0.1.13 → compstats-0.1.14}/PKG-INFO +2 -2
  7. {compstats-0.1.13 → compstats-0.1.14}/pyproject.toml +1 -1
  8. {compstats-0.1.13 → compstats-0.1.14}/CompStats/bootstrap.py +0 -0
  9. {compstats-0.1.13 → compstats-0.1.14}/CompStats/measurements.py +0 -0
  10. {compstats-0.1.13 → compstats-0.1.14}/CompStats/performance.py +0 -0
  11. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/__init__.py +0 -0
  12. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/data.csv +0 -0
  13. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_bootstrap.py +0 -0
  14. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_measurements.py +0 -0
  15. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_metrics.py +0 -0
  16. {compstats-0.1.13 → compstats-0.1.14}/CompStats/tests/test_performance.py +0 -0
  17. {compstats-0.1.13 → compstats-0.1.14}/CompStats/utils.py +0 -0
  18. {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/SOURCES.txt +0 -0
  19. {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/dependency_links.txt +0 -0
  20. {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/requires.txt +0 -0
  21. {compstats-0.1.13 → compstats-0.1.14}/CompStats.egg-info/top_level.txt +0 -0
  22. {compstats-0.1.13 → compstats-0.1.14}/LICENSE +0 -0
  23. {compstats-0.1.13 → compstats-0.1.14}/MANIFEST.in +0 -0
  24. {compstats-0.1.13 → compstats-0.1.14}/README.rst +0 -0
  25. {compstats-0.1.13 → compstats-0.1.14}/setup.cfg +0 -0
  26. {compstats-0.1.13 → compstats-0.1.14}/setup.py +0 -0
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.1.13'
14
+ __version__ = '0.1.14'
15
15
  from CompStats.bootstrap import StatisticSamples
16
16
  from CompStats.measurements import CI, SE, difference_p_value
17
17
  from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -19,7 +19,7 @@ import numpy as np
19
19
  from CompStats.bootstrap import StatisticSamples
20
20
  from CompStats.utils import progress_bar
21
21
  from CompStats import measurements
22
- from CompStats.measurements import SE
22
+ from CompStats.measurements import SE, CI
23
23
  from CompStats.utils import dataframe
24
24
 
25
25
 
@@ -270,7 +270,7 @@ class Perf(object):
270
270
  best = data.argmin()
271
271
  self._best = keys[best]
272
272
  return self._best
273
-
273
+
274
274
  @best.setter
275
275
  def best(self, value):
276
276
  self._best = value
@@ -279,7 +279,7 @@ class Perf(object):
279
279
  def sorting_func(self):
280
280
  """Rank systems when multiple performances are used"""
281
281
  return self._sorting_func
282
-
282
+
283
283
  @sorting_func.setter
284
284
  def sorting_func(self, value):
285
285
  self._sorting_func = value
@@ -315,7 +315,7 @@ class Perf(object):
315
315
  else:
316
316
  self._statistic = dict(data)
317
317
  return self._statistic
318
-
318
+
319
319
  @statistic.setter
320
320
  def statistic(self, value):
321
321
  """statistic setter"""
@@ -346,6 +346,30 @@ class Perf(object):
346
346
  return list(output.values())[0]
347
347
  return output
348
348
 
349
+ @property
350
+ def ci(self):
351
+ """Confidence interval
352
+
353
+ >>> from sklearn.svm import LinearSVC
354
+ >>> from sklearn.datasets import load_iris
355
+ >>> from sklearn.model_selection import train_test_split
356
+ >>> from CompStats.interface import Perf
357
+ >>> X, y = load_iris(return_X_y=True)
358
+ >>> _ = train_test_split(X, y, test_size=0.3)
359
+ >>> X_train, X_val, y_train, y_val = _
360
+ >>> m = LinearSVC().fit(X_train, y_train)
361
+ >>> hy = m.predict(X_val)
362
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
363
+ >>> perf = Perf(y_val, hy, name='LinearSVC')
364
+ >>> perf.ci
365
+ (np.float64(0.9333333333333332), np.float64(1.0))
366
+ """
367
+
368
+ output = CI(self.statistic_samples)
369
+ if len(output) == 1:
370
+ return list(output.values())[0]
371
+ return output
372
+
349
373
  def plot(self, value_name:str=None,
350
374
  var_name:str='Performance',
351
375
  alg_legend:str='Algorithm',
@@ -359,6 +383,7 @@ class Perf(object):
359
383
  winner_legend:str='Best',
360
384
  tie_legend:str='Equivalent',
361
385
  loser_legend:str='Different',
386
+ palette:object=None,
362
387
  **kwargs):
363
388
  """plot with seaborn
364
389
 
@@ -403,10 +428,17 @@ class Perf(object):
403
428
  ci = lambda x: measurements.CI(x, alpha=CI)
404
429
  if comparison:
405
430
  kwargs.update(dict(hue=comp_legend))
431
+ if palette is None:
432
+ pal = sns.color_palette("Paired")
433
+ palette = {winner_legend:pal[1],
434
+ tie_legend:pal[3],
435
+ loser_legend: pal[5]}
406
436
  f_grid = sns.catplot(df, x=value_name, errorbar=ci,
407
437
  y=alg_legend, col=var_name,
408
438
  kind=kind, linestyle=linestyle,
409
- col_wrap=col_wrap, capsize=capsize, **kwargs)
439
+ col_wrap=col_wrap, capsize=capsize,
440
+ palette=palette,
441
+ **kwargs)
410
442
  return f_grid
411
443
 
412
444
  def dataframe(self, comparison:bool=False,
@@ -420,7 +452,22 @@ class Perf(object):
420
452
  tie_legend:str='Equivalent',
421
453
  loser_legend:str='Different',
422
454
  perf_names:str=None):
423
- """Dataframe"""
455
+ """Dataframe
456
+
457
+ >>> from sklearn.svm import LinearSVC
458
+ >>> from sklearn.ensemble import RandomForestClassifier
459
+ >>> from sklearn.datasets import load_iris
460
+ >>> from sklearn.model_selection import train_test_split
461
+ >>> from CompStats.interface import Perf
462
+ >>> X, y = load_iris(return_X_y=True)
463
+ >>> _ = train_test_split(X, y, test_size=0.3)
464
+ >>> X_train, X_val, y_train, y_val = _
465
+ >>> m = LinearSVC().fit(X_train, y_train)
466
+ >>> hy = m.predict(X_val)
467
+ >>> ens = RandomForestClassifier().fit(X_train, y_train)
468
+ >>> perf = Perf(y_val, hy, forest=ens.predict(X_val))
469
+ >>> df = perf.dataframe()
470
+ """
424
471
  if perf_names is None and isinstance(self.best, np.ndarray):
425
472
  func_name = self.statistic_func.__name__
426
473
  perf_names = [f'{func_name}({i})'
@@ -624,7 +671,7 @@ class Difference:
624
671
  return self.statistic[self.best]
625
672
  keys = np.unique(self.best)
626
673
  statistic = np.array([self.statistic[k]
627
- for k in keys])
674
+ for k in keys])
628
675
  m = {v: k for k, v in enumerate(keys)}
629
676
  best = np.array([m[x] for x in self.best])
630
677
  return statistic[best, np.arange(best.shape[0])]
@@ -307,7 +307,7 @@ def macro_f1(y_true, *y_pred, labels=None,
307
307
  sample_weight=None, zero_division='warn',
308
308
  num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
309
309
  **kwargs):
310
- """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_f1`
310
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.f1_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.f1_score`
311
311
 
312
312
  :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
313
313
  :type y_true: numpy.ndarray or pandas.DataFrame
@@ -332,7 +332,7 @@ def macro_recall(y_true, *y_pred, labels=None,
332
332
  sample_weight=None, zero_division='warn',
333
333
  num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
334
334
  **kwargs):
335
- """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_recall`
335
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.recall_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.recall_score`
336
336
 
337
337
  :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
338
338
  :type y_true: numpy.ndarray or pandas.DataFrame
@@ -357,7 +357,7 @@ def macro_precision(y_true, *y_pred, labels=None,
357
357
  sample_weight=None, zero_division='warn',
358
358
  num_samples: int=500, n_jobs: int=-1, use_tqdm=True,
359
359
  **kwargs):
360
- """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.macro_precision`
360
+ """:py:class:`~CompStats.interface.Perf` with :py:func:`~sklearn.metrics.precision_score` (as :py:attr:`score_func`) with the parameteres needed to compute the macro score. The parameters not described can be found in :py:func:`~sklearn.metrics.precision_score`
361
361
 
362
362
  :param y_true: True measurement or could be a pandas.DataFrame where column label 'y' corresponds to the true measurement.
363
363
  :type y_true: numpy.ndarray or pandas.DataFrame
@@ -81,7 +81,7 @@ def test_Perf_dataframe():
81
81
  from CompStats.metrics import f1_score
82
82
 
83
83
  X, y = load_digits(return_X_y=True)
84
- _ = train_test_split(X, y, test_size=0.3)
84
+ _ = train_test_split(X, y, test_size=0.3, random_state=0)
85
85
  X_train, X_val, y_train, y_val = _
86
86
  ens = RandomForestClassifier().fit(X_train, y_train)
87
87
  nb = GaussianNB().fit(X_train, y_train)
@@ -121,6 +121,7 @@ def test_Perf_plot_multi():
121
121
  f_grid = score.plot()
122
122
  assert f_grid is not None
123
123
 
124
+
124
125
  def test_Perf_statistic_one():
125
126
  """Test Perf statistic one alg"""
126
127
  from CompStats.metrics import f1_score
@@ -142,6 +143,9 @@ def test_Perf_statistic_one():
142
143
  assert isinstance(score.statistic, float)
143
144
  assert isinstance(str(score), str)
144
145
  assert isinstance(score.se, float)
146
+ assert isinstance(score.ci, tuple)
147
+ assert len(score.ci) == 2
148
+
145
149
 
146
150
  def test_Perf_best():
147
151
  """Test Perf best"""
@@ -191,7 +195,7 @@ def test_difference_best():
191
195
  score(svm.predict(X_val), name='svm')
192
196
  diff = score.difference()
193
197
  assert isinstance(diff.best, str)
194
-
198
+
195
199
 
196
200
  def test_difference_str__():
197
201
  """Test f1_score"""
@@ -1,7 +1,8 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: CompStats
3
- Version: 0.1.13
3
+ Version: 0.1.14
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
+ License-Expression: Apache-2.0
5
6
  Project-URL: Homepage, https://compstats.readthedocs.io
6
7
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
7
8
  Project-URL: Issues, https://github.com/INGEOTEC/CompStats/issues
@@ -10,7 +11,6 @@ Classifier: Environment :: Console
10
11
  Classifier: Intended Audience :: Developers
11
12
  Classifier: Intended Audience :: Information Technology
12
13
  Classifier: Intended Audience :: Science/Research
13
- Classifier: License :: OSI Approved :: MIT License
14
14
  Classifier: Operating System :: OS Independent
15
15
  Classifier: Programming Language :: Python
16
16
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -1,7 +1,8 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: CompStats
3
- Version: 0.1.13
3
+ Version: 0.1.14
4
4
  Summary: CompStats implements an evaluation methodology for statistically analyzing competition results and competition
5
+ License-Expression: Apache-2.0
5
6
  Project-URL: Homepage, https://compstats.readthedocs.io
6
7
  Project-URL: Repository, https://github.com/INGEOTEC/CompStats
7
8
  Project-URL: Issues, https://github.com/INGEOTEC/CompStats/issues
@@ -10,7 +11,6 @@ Classifier: Environment :: Console
10
11
  Classifier: Intended Audience :: Developers
11
12
  Classifier: Intended Audience :: Information Technology
12
13
  Classifier: Intended Audience :: Science/Research
13
- Classifier: License :: OSI Approved :: MIT License
14
14
  Classifier: Operating System :: OS Independent
15
15
  Classifier: Programming Language :: Python
16
16
  Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
@@ -2,6 +2,7 @@
2
2
  name = 'CompStats'
3
3
  description = 'CompStats implements an evaluation methodology for statistically analyzing competition results and competition'
4
4
  readme = "README.rst"
5
+ license = "Apache-2.0"
5
6
  dependencies = [
6
7
  'numpy',
7
8
  'scikit-learn>=1.3.0',
@@ -17,7 +18,6 @@ classifiers = [
17
18
  "Intended Audience :: Developers",
18
19
  "Intended Audience :: Information Technology",
19
20
  "Intended Audience :: Science/Research",
20
- "License :: OSI Approved :: MIT License",
21
21
  "Operating System :: OS Independent",
22
22
  "Programming Language :: Python",
23
23
  "Topic :: Scientific/Engineering :: Artificial Intelligence",
File without changes
File without changes
File without changes
File without changes
File without changes