CompStats 0.0.3__tar.gz → 0.0.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,7 @@
11
11
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
- __version__ = '0.0.3'
14
+ __version__ = '0.0.5'
15
15
  from CompStats.bootstrap import StatisticSamples
16
- from CompStats.measurements import CI
17
- from CompStats.performance import performance, difference, plot_performance, plot_difference
16
+ from CompStats.measurements import CI, difference_p_value
17
+ from CompStats.performance import performance, difference, all_differences, plot_performance, plot_difference
@@ -12,7 +12,8 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import numpy as np
15
-
15
+ import pandas as pd
16
+ from CompStats.bootstrap import StatisticSamples
16
17
 
17
18
  def CI(samples: np.ndarray, alpha=0.05):
18
19
  """Compute the Confidence Interval of a statistic using bootstrap.
@@ -34,3 +35,9 @@ def CI(samples: np.ndarray, alpha=0.05):
34
35
  alpha = alpha / 2
35
36
  return (np.percentile(samples, alpha * 100, axis=0),
36
37
  np.percentile(samples, (1 - alpha) * 100, axis=0))
38
+
39
+
40
+ def difference_p_value(statistic_samples: StatisticSamples):
41
+ """Compute the difference p-value"""
42
+ return {k: (v > 2 * np.mean(v)).mean()
43
+ for k, v in statistic_samples.calls.items()}
@@ -24,10 +24,13 @@ from CompStats.measurements import CI
24
24
  def performance(data: pd.DataFrame,
25
25
  gold: str='y',
26
26
  score: Callable[[np.ndarray, np.ndarray], float]=accuracy_score,
27
+ num_samples: int=500,
28
+ n_jobs: int=-1,
27
29
  statistic_samples: StatisticSamples=None) -> StatisticSamples:
28
30
  """Bootstrap samples of a performance score"""
29
31
  if statistic_samples is None:
30
- statistic_samples = StatisticSamples(statistic=score)
32
+ statistic_samples = StatisticSamples(statistic=score, num_samples=num_samples,
33
+ n_jobs=n_jobs)
31
34
  columns = data.columns
32
35
  y = data[gold]
33
36
  for column in columns:
@@ -37,7 +40,7 @@ def performance(data: pd.DataFrame,
37
40
  return statistic_samples
38
41
 
39
42
 
40
- def difference(statistic_samples: StatisticSamples, best_index=-1):
43
+ def difference(statistic_samples: StatisticSamples, best_index: int=-1):
41
44
  """Bootstrap samples of a difference in performnace"""
42
45
 
43
46
  items = list(statistic_samples.calls.items())
@@ -53,6 +56,30 @@ def difference(statistic_samples: StatisticSamples, best_index=-1):
53
56
  output.calls = diff
54
57
  output.info['best'] = best_name
55
58
  return output
59
+
60
+
61
+ def all_differences(statistic_samples: StatisticSamples, reverse: bool=True):
62
+ """Calculates all possible differences in performance among algorithms and sorts by average performance"""
63
+
64
+ items = list(statistic_samples.calls.items())
65
+ # Calculamos el rendimiento medio y ordenamos los algoritmos basándonos en este
66
+ perf = [(k, v, np.mean(v)) for k, v in items]
67
+ perf.sort(key=lambda x: x[2], reverse=reverse) # Orden descendente por rendimiento medio
68
+
69
+ diffs = {} # Diccionario para guardar las diferencias
70
+
71
+ # Iteramos sobre todos los pares posibles de algoritmos ordenados
72
+ for i in range(len(perf)):
73
+ for j in range(i + 1, len(perf)):
74
+ name_i, perf_i, _ = perf[i]
75
+ name_j, perf_j, _ = perf[j]
76
+
77
+ # Diferencia de i a j
78
+ diff_key_i_to_j = f"{name_i} - {name_j}"
79
+ diffs[diff_key_i_to_j] = np.array(perf_i) - np.array(perf_j)
80
+ output = clone(statistic_samples)
81
+ output.calls = diffs
82
+ return output
56
83
 
57
84
 
58
85
  def plot_performance(statistic_samples: StatisticSamples,
@@ -12,9 +12,14 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
  import numpy as np
15
- from CompStats.measurements import CI
15
+ import pandas as pd
16
+ import os
17
+ from sklearn.metrics import f1_score
18
+ from CompStats.measurements import CI, difference_p_value
16
19
  from CompStats.bootstrap import StatisticSamples
20
+ from CompStats.performance import performance, difference
17
21
 
22
+ DATA = os.path.join(os.path.dirname(__file__), 'data.csv')
18
23
 
19
24
  def test_CI():
20
25
  """Test confidence interval"""
@@ -24,4 +29,13 @@ def test_CI():
24
29
  samples = statistic(pop)
25
30
  low, high = CI(samples)
26
31
  mean = pop.mean()
27
- assert low < mean < high
32
+ assert low < mean < high
33
+
34
+
35
+ def test_difference_p_value():
36
+ """Test difference p-values"""
37
+ df = pd.read_csv(DATA)
38
+ perf = performance(df, score=lambda y, hy: f1_score(y, hy, average='weighted'))
39
+ res = difference(perf)
40
+ p_value = difference_p_value(res)
41
+ assert p_value['BoW'] > 0.2
@@ -16,7 +16,7 @@ import pandas as pd
16
16
  import os
17
17
  from sklearn.metrics import f1_score
18
18
  import seaborn as sns
19
- from CompStats.performance import performance, plot_performance, difference, plot_difference
19
+ from CompStats.performance import performance, plot_performance, difference, plot_difference, all_differences
20
20
 
21
21
 
22
22
  DATA = os.path.join(os.path.dirname(__file__), 'data.csv')
@@ -27,6 +27,7 @@ def test_performance():
27
27
  perf = performance(df, score=lambda y, hy: f1_score(y, hy, average='weighted'))
28
28
  assert 'BoW' in perf.calls
29
29
  assert 'y' not in perf.calls
30
+ assert perf.n_jobs == -1
30
31
 
31
32
 
32
33
  def test_plot_performance():
@@ -44,4 +45,12 @@ def test_difference():
44
45
  assert 'INGEOTEC' not in res.calls
45
46
  assert res.info['best'] == 'INGEOTEC'
46
47
  ins = plot_difference(res)
47
- assert isinstance(ins, sns.FacetGrid)
48
+ assert isinstance(ins, sns.FacetGrid)
49
+
50
+
51
+ def test_all_differences():
52
+ """Test all_differences"""
53
+ df = pd.read_csv(DATA)
54
+ perf = performance(df, score=lambda y, hy: f1_score(y, hy, average='weighted'))
55
+ res = all_differences(perf)
56
+ assert 'INGEOTEC - BoW' in res.calls
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: CompStats
3
- Version: 0.0.3
3
+ Version: 0.0.5
4
4
  License-File: LICENSE
5
5
  Requires-Dist: numpy
6
6
  Requires-Dist: scikit-learn>=1.3.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: CompStats
3
- Version: 0.0.3
3
+ Version: 0.0.5
4
4
  License-File: LICENSE
5
5
  Requires-Dist: numpy
6
6
  Requires-Dist: scikit-learn>=1.3.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes