khalib 0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
khalib-0.2/LICENSE.md ADDED
@@ -0,0 +1,28 @@
1
+ BSD 3-Clause License
2
+
3
+ Copyright (c) 2025 The khalib contributors.
4
+
5
+ Redistribution and use in source and binary forms, with or without
6
+ modification, are permitted provided that the following conditions are met:
7
+
8
+ 1. Redistributions of source code must retain the above copyright notice, this
9
+ list of conditions and the following disclaimer.
10
+
11
+ 2. Redistributions in binary form must reproduce the above copyright notice,
12
+ this list of conditions and the following disclaimer in the documentation
13
+ and/or other materials provided with the distribution.
14
+
15
+ 3. Neither the name of the copyright holder nor the names of its
16
+ contributors may be used to endorse or promote products derived from
17
+ this software without specific prior written permission.
18
+
19
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
khalib-0.2/PKG-INFO ADDED
@@ -0,0 +1,76 @@
1
+ Metadata-Version: 2.4
2
+ Name: khalib
3
+ Version: 0.2
4
+ Summary: Classifier Calibration with Khiops
5
+ Author-email: Felipe Olmos <luisfelipe.olmosmarchant@orange.com>
6
+ License-Expression: BSD-3-Clause
7
+ Project-URL: homepage, https://khiopslab.github.io/khalib
8
+ Project-URL: source, https://github.com/KhiopsLab/khalib
9
+ Project-URL: tracker, https://github.com/KhiopsLab/khalib/issues
10
+ Project-URL: changelog, https://github.com/KhiopsLab/khalib/blob/main/CHANGELOG.md
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python
14
+ Classifier: Topic :: Software Development
15
+ Classifier: Topic :: Scientific/Engineering
16
+ Classifier: Development Status :: 5 - Production/Stable
17
+ Classifier: Operating System :: Microsoft :: Windows
18
+ Classifier: Operating System :: POSIX
19
+ Classifier: Operating System :: Unix
20
+ Classifier: Operating System :: MacOS
21
+ Classifier: Programming Language :: Python :: 3
22
+ Classifier: Programming Language :: Python :: 3.11
23
+ Classifier: Programming Language :: Python :: 3.12
24
+ Classifier: Programming Language :: Python :: 3.13
25
+ Classifier: Programming Language :: Python :: 3.14
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE.md
29
+ Requires-Dist: khiops>=11.0.0.0
30
+ Requires-Dist: numpy>=2.0.0
31
+ Requires-Dist: scikit-learn>=1.5.0
32
+ Requires-Dist: matplotlib>=3.9.0
33
+ Dynamic: license-file
34
+
35
+ <!-- start-summary -->
36
+ # khalib
37
+ `khalib` is a classifier probability calibration package powered by the [Khiops][khiops-org] AutoML
38
+ suite.
39
+
40
+
41
+ ## Features
42
+ - `KhalibClassifier`: A scikit-learn estimator to calibrate classifiers with a similar interface
43
+ as [CalibratedClassifierCV][sk-calclf].
44
+ - `calibration_error` : A function to estimate the Estimated Calibration Error (ECE).
45
+ - `build_reliability_diagram` : A function to build a reliability diagram.
46
+
47
+ These features are based on Khiops's non-parametric supervised histograms, so there is no need to
48
+ specify the number and width of the bins, as they are automatically estimated from data.
49
+
50
+ [khiops-org]: https://khiops.org
51
+ [sk-calclf]: https://scikit-learn.org/stable/modules/generated/sklearn.calibration.CalibratedClassifierCV.html
52
+
53
+ <!-- end-summary -->
54
+
55
+ See the [documentation][khalib-docs] for more information.
56
+
57
+ <!-- start-install -->
58
+ ## Installation
59
+
60
+ - [Install Khiops][khiops-setup]
61
+ - Install the `khalib` python library:
62
+
63
+ ```sh
64
+ pip install khalib
65
+ ```
66
+ [khiops-setup]: https://khiops.org/setup/
67
+
68
+ <!-- end-install -->
69
+
70
+
71
+ ## Documentation
72
+
73
+ See https://khiopslab.github.io/khalib/
74
+
75
+
76
+ [khalib-docs]: https://khiopslab.github.io/khalib
khalib-0.2/README.md ADDED
@@ -0,0 +1,42 @@
1
+ <!-- start-summary -->
2
+ # khalib
3
+ `khalib` is a classifier probability calibration package powered by the [Khiops][khiops-org] AutoML
4
+ suite.
5
+
6
+
7
+ ## Features
8
+ - `KhalibClassifier`: A scikit-learn estimator to calibrate classifiers with a similar interface
9
+ as [CalibratedClassifierCV][sk-calclf].
10
+ - `calibration_error` : A function to estimate the Estimated Calibration Error (ECE).
11
+ - `build_reliability_diagram` : A function to build a reliability diagram.
12
+
13
+ These features are based on Khiops's non-parametric supervised histograms, so there is no need to
14
+ specify the number and width of the bins, as they are automatically estimated from data.
15
+
16
+ [khiops-org]: https://khiops.org
17
+ [sk-calclf]: https://scikit-learn.org/stable/modules/generated/sklearn.calibration.CalibratedClassifierCV.html
18
+
19
+ <!-- end-summary -->
20
+
21
+ See the [documentation][khalib-docs] for more information.
22
+
23
+ <!-- start-install -->
24
+ ## Installation
25
+
26
+ - [Install Khiops][khiops-setup]
27
+ - Install the `khalib` python library:
28
+
29
+ ```sh
30
+ pip install khalib
31
+ ```
32
+ [khiops-setup]: https://khiops.org/setup/
33
+
34
+ <!-- end-install -->
35
+
36
+
37
+ ## Documentation
38
+
39
+ See https://khiopslab.github.io/khalib/
40
+
41
+
42
+ [khalib-docs]: https://khiopslab.github.io/khalib
@@ -0,0 +1,102 @@
1
+ [build-system]
2
+ requires = ["setuptools", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "khalib"
7
+ version = "0.2"
8
+ description = "Classifier Calibration with Khiops"
9
+ authors = [{ name = "Felipe Olmos", email = "luisfelipe.olmosmarchant@orange.com" }]
10
+ requires-python = ">=3.11"
11
+ dependencies = [
12
+ "khiops>=11.0.0.0",
13
+ "numpy>=2.0.0",
14
+ "scikit-learn>=1.5.0",
15
+ "matplotlib>=3.9.0",
16
+ ]
17
+ readme = "README.md"
18
+ license = "BSD-3-Clause"
19
+ license-files = ["LICENSE.md"]
20
+ classifiers=[
21
+ "Intended Audience :: Science/Research",
22
+ "Intended Audience :: Developers",
23
+ "Programming Language :: Python",
24
+ "Topic :: Software Development",
25
+ "Topic :: Scientific/Engineering",
26
+ "Development Status :: 5 - Production/Stable",
27
+ "Operating System :: Microsoft :: Windows",
28
+ "Operating System :: POSIX",
29
+ "Operating System :: Unix",
30
+ "Operating System :: MacOS",
31
+ "Programming Language :: Python :: 3",
32
+ "Programming Language :: Python :: 3.11",
33
+ "Programming Language :: Python :: 3.12",
34
+ "Programming Language :: Python :: 3.13",
35
+ "Programming Language :: Python :: 3.14",
36
+ ]
37
+ [project.urls]
38
+ homepage = "https://khiopslab.github.io/khalib"
39
+ source = "https://github.com/KhiopsLab/khalib"
40
+ tracker = "https://github.com/KhiopsLab/khalib/issues"
41
+ changelog = "https://github.com/KhiopsLab/khalib/blob/main/CHANGELOG.md"
42
+
43
+
44
+ [tool.setuptools.packages.find]
45
+ where = ["src"]
46
+
47
+ [tool.pytest.ini_options]
48
+ markers = [
49
+ "slow: marks tests as slow (deselect with '-m \"not slow\"')",
50
+ ]
51
+
52
+ [tool.ruff.lint]
53
+ exclude = ["docs/quickstart.py"]
54
+ select = [
55
+ "E", # pycodestyle errors
56
+ "F", # pyflakes
57
+ "I", # isort
58
+ "B", # flake8-bugbear
59
+ "UP", # pyupgrade
60
+ "N", # pep8-naming
61
+ "NPY", # numpy
62
+ "PD", # pandas-vet
63
+ "C4", # flake8-comprehensions
64
+ "SIM", # flake8-simplify
65
+ "TID", # tidy imports
66
+ "PT", # flake8-pytest-style
67
+ "PL", # pylint rules
68
+ ]
69
+ ignore = [
70
+ "PLR0904", # too-many-public-methods
71
+ "PLR0911", # too-many-return-statements
72
+ "PLR0912", # too-many-branches
73
+ "PLR0913", # too-many-arguments
74
+ "PLR0914", # too-many-locals
75
+ "PLR0915", # too-many-statements
76
+ "PLR0916", # too-many-boolean-expressions
77
+ "PLR1702", # too-many-nested-blocks
78
+ "PLR2004", # magic-value-comparison
79
+ "PLW0120", # useless-else-on-loop
80
+ "PLW0406", # import-self
81
+ "PLW0603", # global-statement
82
+ "PD901", # Avoid 'df' name for dataframes
83
+ ]
84
+ # To fix by default isort fixes
85
+ extend-safe-fixes = ["I"]
86
+
87
+ [dependency-groups]
88
+ dev = [
89
+ "furo>=2025.7.19",
90
+ "ipython>=8.27.0",
91
+ "jupyterlab>=4.4.9",
92
+ "jupyterlab-myst>=2.4.2",
93
+ "jupytext>=1.17.3",
94
+ "matplotlib>=3.10.6",
95
+ "myst-nb>=1.3.0",
96
+ "myst-parser>=4.0.1",
97
+ "numpydoc>=1.9.0",
98
+ "pytest>=8.4.1",
99
+ "sphinx>=8.2.3",
100
+ "sphinx-copybutton>=0.5.2",
101
+ "sphinx-paramlinks>=0.6.0",
102
+ ]
khalib-0.2/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,76 @@
1
+ Metadata-Version: 2.4
2
+ Name: khalib
3
+ Version: 0.2
4
+ Summary: Classifier Calibration with Khiops
5
+ Author-email: Felipe Olmos <luisfelipe.olmosmarchant@orange.com>
6
+ License-Expression: BSD-3-Clause
7
+ Project-URL: homepage, https://khiopslab.github.io/khalib
8
+ Project-URL: source, https://github.com/KhiopsLab/khalib
9
+ Project-URL: tracker, https://github.com/KhiopsLab/khalib/issues
10
+ Project-URL: changelog, https://github.com/KhiopsLab/khalib/blob/main/CHANGELOG.md
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python
14
+ Classifier: Topic :: Software Development
15
+ Classifier: Topic :: Scientific/Engineering
16
+ Classifier: Development Status :: 5 - Production/Stable
17
+ Classifier: Operating System :: Microsoft :: Windows
18
+ Classifier: Operating System :: POSIX
19
+ Classifier: Operating System :: Unix
20
+ Classifier: Operating System :: MacOS
21
+ Classifier: Programming Language :: Python :: 3
22
+ Classifier: Programming Language :: Python :: 3.11
23
+ Classifier: Programming Language :: Python :: 3.12
24
+ Classifier: Programming Language :: Python :: 3.13
25
+ Classifier: Programming Language :: Python :: 3.14
26
+ Requires-Python: >=3.11
27
+ Description-Content-Type: text/markdown
28
+ License-File: LICENSE.md
29
+ Requires-Dist: khiops>=11.0.0.0
30
+ Requires-Dist: numpy>=2.0.0
31
+ Requires-Dist: scikit-learn>=1.5.0
32
+ Requires-Dist: matplotlib>=3.9.0
33
+ Dynamic: license-file
34
+
35
+ <!-- start-summary -->
36
+ # khalib
37
+ `khalib` is a classifier probability calibration package powered by the [Khiops][khiops-org] AutoML
38
+ suite.
39
+
40
+
41
+ ## Features
42
+ - `KhalibClassifier`: A scikit-learn estimator to calibrate classifiers with a similar interface
43
+ as [CalibratedClassifierCV][sk-calclf].
44
+ - `calibration_error` : A function to estimate the Estimated Calibration Error (ECE).
45
+ - `build_reliability_diagram` : A function to build a reliability diagram.
46
+
47
+ These features are based on Khiops's non-parametric supervised histograms, so there is no need to
48
+ specify the number and width of the bins, as they are automatically estimated from data.
49
+
50
+ [khiops-org]: https://khiops.org
51
+ [sk-calclf]: https://scikit-learn.org/stable/modules/generated/sklearn.calibration.CalibratedClassifierCV.html
52
+
53
+ <!-- end-summary -->
54
+
55
+ See the [documentation][khalib-docs] for more information.
56
+
57
+ <!-- start-install -->
58
+ ## Installation
59
+
60
+ - [Install Khiops][khiops-setup]
61
+ - Install the `khalib` python library:
62
+
63
+ ```sh
64
+ pip install khalib
65
+ ```
66
+ [khiops-setup]: https://khiops.org/setup/
67
+
68
+ <!-- end-install -->
69
+
70
+
71
+ ## Documentation
72
+
73
+ See https://khiopslab.github.io/khalib/
74
+
75
+
76
+ [khalib-docs]: https://khiopslab.github.io/khalib
@@ -0,0 +1,9 @@
1
+ LICENSE.md
2
+ README.md
3
+ pyproject.toml
4
+ src/khalib.egg-info/PKG-INFO
5
+ src/khalib.egg-info/SOURCES.txt
6
+ src/khalib.egg-info/dependency_links.txt
7
+ src/khalib.egg-info/requires.txt
8
+ src/khalib.egg-info/top_level.txt
9
+ src/tests/test_all.py
@@ -0,0 +1,4 @@
1
+ khiops>=11.0.0.0
2
+ numpy>=2.0.0
3
+ scikit-learn>=1.5.0
4
+ matplotlib>=3.9.0
@@ -0,0 +1 @@
1
+ tests
@@ -0,0 +1,445 @@
1
+ import json
2
+ import os
3
+
4
+ import numpy as np
5
+ import pandas as pd
6
+ import pytest
7
+ from khiops.sklearn import KhiopsClassifier
8
+ from sklearn.datasets import make_classification
9
+ from sklearn.linear_model import LogisticRegression
10
+ from sklearn.model_selection import train_test_split
11
+ from sklearn.svm import SVC
12
+ from sklearn.utils.validation import check_is_fitted
13
+
14
+ from khalib import Histogram, KhalibClassifier, calibration_error
15
+
16
+
17
+ @pytest.fixture(name="data_root_dir")
18
+ def fixture_data_root_dir():
19
+ return os.path.join(os.path.dirname(__file__), "data")
20
+
21
+
22
+ @pytest.fixture(name="adult_scores_df")
23
+ def fixture_adult_scores_df(data_root_dir):
24
+ return pd.read_csv(f"{data_root_dir}/tables/adult_scores_sample.tsv", sep="\t")
25
+
26
+
27
+ @pytest.fixture(name="y_fixtures")
28
+ def fixture_y_fixtures(adult_scores_df):
29
+ rng = np.random.default_rng(seed=1234567)
30
+ return {
31
+ "int": adult_scores_df.y,
32
+ "str": np.where(adult_scores_df.y == 0, "zero", "one"),
33
+ "intnl": np.where(adult_scores_df.y == 0, 10, 2),
34
+ "bool": np.where(adult_scores_df.y == 0, False, True),
35
+ "float": np.where(adult_scores_df.y == 0, 10.1, 2.1),
36
+ "random": rng.integers(low=0, high=2, size=adult_scores_df.shape[0]),
37
+ }
38
+
39
+
40
+ @pytest.fixture(name="y_scores_fixtures")
41
+ def fixture_y_scores_fixtures(adult_scores_df):
42
+ y_scores = adult_scores_df.y_score
43
+ return {
44
+ "original": y_scores,
45
+ "original-2d": np.transpose(np.vstack([1 - y_scores, y_scores])),
46
+ "original-fake-2d": np.array(y_scores).reshape(y_scores.shape[0], 1),
47
+ "constant": np.full(y_scores.shape[0], 0.5),
48
+ }
49
+
50
+
51
+ @pytest.fixture(name="vehicles_scores_df")
52
+ def fixture_vehicles_scores_df(data_root_dir):
53
+ return pd.read_csv(f"{data_root_dir}/tables/vehicles_scores.tsv", sep="\t")
54
+
55
+
56
+ @pytest.fixture(name="short_test_id")
57
+ def fixture_data_df(request):
58
+ trans_table = str.maketrans("[", "_", "]")
59
+ return request.node.nodeid.split("::")[-1].translate(trans_table)
60
+
61
+
62
+ @pytest.fixture(name="ref_histogram")
63
+ def fixture_ref_histogram(data_root_dir, short_test_id):
64
+ with open(f"{data_root_dir}/histogram/ref/{short_test_id}.json") as ref_json_file:
65
+ yield read_histogram_from_json_data(json.load(ref_json_file))
66
+
67
+
68
+ def read_histogram_from_json_data(json_data):
69
+ return Histogram(
70
+ breakpoints=json_data["breakpoints"],
71
+ freqs=json_data["freqs"],
72
+ target_freqs=[tuple(cur_freqs) for cur_freqs in json_data["target_freqs"]],
73
+ classes=json_data["classes"],
74
+ )
75
+
76
+
77
+ def is_target_inverted(y_fixture):
78
+ return y_fixture in ["float", "intnl", "str"]
79
+
80
+
81
+ class TestHistogram:
82
+ all_cases = [
83
+ ("eq-freq", "bool", True),
84
+ ("eq-freq", "float", True),
85
+ ("eq-freq", "int", False),
86
+ ("eq-freq", "int", True),
87
+ ("eq-freq", "intnl", True),
88
+ ("eq-width", "bool", True),
89
+ ("eq-width", "float", True),
90
+ ("eq-width", "int", False),
91
+ ("eq-width", "int", True),
92
+ ("eq-width", "intnl", True),
93
+ ("khiops", "bool", True),
94
+ ("khiops", "float", True),
95
+ ("khiops", "int", False),
96
+ ("khiops", "int", True),
97
+ ("khiops", "intnl", True),
98
+ ]
99
+
100
+ @pytest.mark.parametrize(("method", "y_fixture", "use_y"), all_cases)
101
+ def test_happy_path(
102
+ self, y_fixtures, y_scores_fixtures, ref_histogram, method, y_fixture, use_y
103
+ ):
104
+ for y_scores_fixture in ["original", "original-fake-2d"]:
105
+ # Prepare the input data for the histogram
106
+ y = y_fixtures[y_fixture] if use_y else None
107
+ y_scores = y_scores_fixtures[y_scores_fixture]
108
+
109
+ # Compute the histogram with the test settings, check it against the
110
+ # reference
111
+ histogram = Histogram.from_data(y_scores, y=y, method=method)
112
+ assert histogram == ref_histogram, f"Fixture '{y_scores_fixture}'"
113
+
114
+ @pytest.mark.parametrize(("method", "y_fixture", "use_y"), all_cases)
115
+ def test_single_value_score(
116
+ self, y_fixtures, y_scores_fixtures, ref_histogram, method, y_fixture, use_y
117
+ ):
118
+ # Prepare the input data for the histogram
119
+ y = y_fixtures[y_fixture] if use_y else None
120
+ y_scores = y_scores_fixtures["constant"]
121
+
122
+ # Compute the histogram with the test settings, check it against the reference
123
+ histogram = Histogram.from_data(y_scores, y=y, method=method)
124
+ assert histogram == ref_histogram
125
+
126
+ @pytest.mark.parametrize("method", ["eq-freq", "eq-width", "khiops"])
127
+ def test_no_info_target(self, y_fixtures, y_scores_fixtures, ref_histogram, method):
128
+ # Prepare the input data for the histogram
129
+ y = y_fixtures["random"]
130
+ y_scores = y_scores_fixtures["original"]
131
+
132
+ # Compute the histogram with the test settings, check it against the reference
133
+ histogram = Histogram.from_data(y_scores, y=y, method=method)
134
+ assert histogram == ref_histogram
135
+
136
+ def test_find_vfind_coherence(self, y_fixtures, y_scores_fixtures):
137
+ # Prepare the input data for the histogram
138
+ y = y_fixtures["int"]
139
+ y_scores = y_scores_fixtures["original"]
140
+
141
+ # Create an equal width histogram
142
+ histogram = Histogram.from_data(y_scores, y=y, method="eq-width")
143
+
144
+ # Check the coherence for some test points
145
+ test_scores = [i / 10 for i in range(-1, 12)]
146
+ np.testing.assert_array_equal(
147
+ histogram.vfind(test_scores),
148
+ [histogram.find(score) for score in test_scores],
149
+ )
150
+
151
+ # Check the coherence for the histogram breakpoints
152
+ np.testing.assert_array_equal(
153
+ histogram.vfind(histogram.breakpoints),
154
+ [histogram.find(score) for score in histogram.breakpoints],
155
+ )
156
+
157
+ @pytest.mark.parametrize("y_scores_fixture", ["original", "constant"])
158
+ @pytest.mark.parametrize("method", ["eq-freq", "eq-width", "khiops"])
159
+ def test_manual_vs_khiops_coherence(
160
+ self, y_fixtures, y_scores_fixture, y_scores_fixtures, method
161
+ ):
162
+ # Prepare the input data for the histogram
163
+ y = y_fixtures["int"]
164
+ y_scores = y_scores_fixtures[y_scores_fixture]
165
+
166
+ # Obtain the histogram first with Khiops
167
+ histogram_ref = Histogram.from_data(y_scores, y=y, method=method)
168
+
169
+ # Rebuild it with the `from_data_and_breakpoints` method and check
170
+ histogram = Histogram.from_data_and_breakpoints(
171
+ y_scores, histogram_ref.breakpoints, y=y
172
+ )
173
+ assert histogram == histogram_ref
174
+
175
+
176
+ def sigmoid_stair(x):
177
+ return (
178
+ 0.2 / (1 + np.exp(-400 * (x - 0.11)))
179
+ + 0.3 / (1 + np.exp(-400 * (x - 0.19)))
180
+ + 0.1 / (1 + np.exp(-400 * (x - 0.39)))
181
+ + 0.25 / (1 + np.exp(-400 * (x - 0.75)))
182
+ + 0.15 / (1 + np.exp(-400 * (x - 0.91)))
183
+ )
184
+
185
+
186
+ def sine_cube_root(x):
187
+ return 0.5 * (np.power(x, 1 / 3) * (1 + np.cos(16 * np.pi * x)))
188
+
189
+
190
+ def identity(x):
191
+ return x
192
+
193
+
194
+ @pytest.fixture(name="distorted_scores")
195
+ def fixture_distorted_scores():
196
+ # Create the random scores
197
+ rng = np.random.default_rng(seed=1234567)
198
+ y_scores = rng.beta(a=0.4, b=0.6, size=100_000)
199
+ y_scores_fixtures = {
200
+ "1d": y_scores,
201
+ "2d": np.transpose(np.vstack([1 - y_scores, y_scores])),
202
+ "fake-2d": np.array(y_scores).reshape(y_scores.shape[0], 1),
203
+ }
204
+ # Build the fixture data dictionary
205
+ data = {}
206
+ data["y_scores_fixtures"] = y_scores_fixtures
207
+ for distortion_fun in (sigmoid_stair, sine_cube_root, identity):
208
+ # Sample a binary distribution according to the distorted probabilities
209
+ distortion_data = {}
210
+ y_cond_probas = distortion_fun(y_scores)
211
+ y = rng.binomial(1, y_cond_probas)
212
+
213
+ # Store the distorted sample and its variants
214
+ distortion_data["y_fixtures"] = {
215
+ "int": (y, False),
216
+ "str": (np.where(y == 0, "zero", "one"), True),
217
+ }
218
+
219
+ # Store the expecte ECE and error margin for each distortion function
220
+ # Note: These were calculated with a grid of 1e-8 intervals
221
+ if distortion_fun.__name__ == "sine_cube_root":
222
+ distortion_data |= {
223
+ "theoretical_ece": 0.2381850813972994,
224
+ "approx_kwargs": {"abs": 1.0e-3},
225
+ }
226
+ elif distortion_fun.__name__ == "sigmoid_stair":
227
+ distortion_data |= {
228
+ "theoretical_ece": 0.07390433812747241,
229
+ "approx_kwargs": {"abs": 1.0e-3},
230
+ }
231
+ else:
232
+ assert distortion_fun.__name__ == "identity"
233
+ distortion_data |= {"theoretical_ece": 0, "approx_kwargs": {"abs": 2.5e-2}}
234
+ data[distortion_fun.__name__] = distortion_data
235
+
236
+ return data
237
+
238
+
239
+ class TestECE:
240
+ @pytest.mark.parametrize("y_fixture", ["bool", "float", "int", "intnl", "str"])
241
+ @pytest.mark.parametrize("y_scores_fixture", ["original", "original-2d"])
242
+ @pytest.mark.parametrize(
243
+ ("method", "expected_ece"), [("bin", 0.036162213), ("label-bin", 0.086438357)]
244
+ )
245
+ def test_fast_binary_ece(
246
+ self,
247
+ method,
248
+ expected_ece,
249
+ y_fixture,
250
+ y_fixtures,
251
+ y_scores_fixture,
252
+ y_scores_fixtures,
253
+ ):
254
+ # Prepare the input data for the ECE estimation
255
+ y = y_fixtures[y_fixture]
256
+ y_scores = y_scores_fixtures[y_scores_fixture].copy()
257
+ if is_target_inverted(y_fixture):
258
+ if len(y_scores.shape) == 1:
259
+ y_scores = 1 - y_scores
260
+ else:
261
+ y_scores[:, [0, 1]] = y_scores[:, [1, 0]]
262
+
263
+ # Estimate the ECE
264
+ ece = calibration_error(y_scores, y, method=method)
265
+ assert ece == pytest.approx(expected_ece)
266
+
267
+ @pytest.mark.parametrize(
268
+ ("multi_class_method", "expected_ece"),
269
+ [("top-label", 0.0723944), ("classwise", 0.04642129)],
270
+ )
271
+ def test_fast_multi_class_ece(
272
+ self, multi_class_method, expected_ece, vehicles_scores_df
273
+ ):
274
+ ece = calibration_error(
275
+ vehicles_scores_df.drop("y", axis=1).__array__(),
276
+ vehicles_scores_df.y,
277
+ multi_class_method=multi_class_method,
278
+ )
279
+ assert ece == pytest.approx(expected_ece)
280
+
281
+ @pytest.mark.slow
282
+ @pytest.mark.parametrize("y_fixture_name", ["int", "str"])
283
+ @pytest.mark.parametrize("y_scores_fixture_name", ["1d", "2d", "fake-2d"])
284
+ @pytest.mark.parametrize(
285
+ "distortion_fun_name", ["sine_cube_root", "sigmoid_stair", "identity"]
286
+ )
287
+ @pytest.mark.parametrize(
288
+ ("method", "expected_ece"), [("label-bin", 0)]
289
+ ) # [("bin", 0.036162213), ("label-bin", 0.086438357)]
290
+ # )
291
+ def test_accuracy_binary_ece(
292
+ self,
293
+ method,
294
+ expected_ece,
295
+ distorted_scores,
296
+ distortion_fun_name,
297
+ y_scores_fixture_name,
298
+ y_fixture_name,
299
+ ):
300
+ # Prepare the input data for the ECE estimation
301
+ y_scores = distorted_scores["y_scores_fixtures"][y_scores_fixture_name].copy()
302
+ y, inverted_target = distorted_scores[distortion_fun_name]["y_fixtures"][
303
+ y_fixture_name
304
+ ]
305
+ if inverted_target:
306
+ if len(y_scores.shape) == 1 or y_scores.shape[1] == 1:
307
+ y_scores = 1 - y_scores
308
+ else:
309
+ y_scores[:, [0, 1]] = y_scores[:, [1, 0]]
310
+
311
+ # Estimate the ECE
312
+ ece = calibration_error(y_scores, y, method=method)
313
+ assert ece == pytest.approx(
314
+ distorted_scores[distortion_fun_name]["theoretical_ece"],
315
+ **distorted_scores[distortion_fun_name]["approx_kwargs"],
316
+ )
317
+
318
+
319
+ def _clf_data_builder(n_train, n_calib, n_test, n_classes, n_features, n_informative):
320
+ # Note, we build this way to ensure reproducibility for the not_fitted/None case
321
+ x, y = make_classification(
322
+ n_samples=n_train + n_calib + n_test,
323
+ n_classes=n_classes,
324
+ n_features=n_features,
325
+ n_informative=n_informative,
326
+ random_state=1,
327
+ return_X_y=True,
328
+ )
329
+ x_train, x_not_train, y_train, y_not_train = train_test_split(
330
+ x, y, train_size=n_train, random_state=2
331
+ )
332
+ x_calib, x_test, y_calib, y_test = train_test_split(
333
+ x_not_train, y_not_train, train_size=n_calib, random_state=3
334
+ )
335
+
336
+ return x_train, x_calib, x_test, y_train, y_calib, y_test
337
+
338
+
339
+ @pytest.fixture(name="clf_data", scope="session")
340
+ def fixture_clf_data():
341
+ return {
342
+ "binary": {
343
+ "slow": _clf_data_builder(
344
+ n_train=10_000,
345
+ n_calib=10_000,
346
+ n_test=30_000,
347
+ n_classes=2,
348
+ n_features=10,
349
+ n_informative=2,
350
+ ),
351
+ "fast": _clf_data_builder(
352
+ n_train=1_000,
353
+ n_calib=1_000,
354
+ n_test=1_000,
355
+ n_classes=2,
356
+ n_features=10,
357
+ n_informative=2,
358
+ ),
359
+ },
360
+ "multi_class": {
361
+ "slow": _clf_data_builder(
362
+ n_train=10_000,
363
+ n_calib=10_000,
364
+ n_test=30_000,
365
+ n_classes=4,
366
+ n_features=10,
367
+ n_informative=4,
368
+ ),
369
+ "fast": _clf_data_builder(
370
+ n_train=1_000,
371
+ n_calib=1_000,
372
+ n_test=1_000,
373
+ n_classes=4,
374
+ n_features=10,
375
+ n_informative=4,
376
+ ),
377
+ },
378
+ }
379
+
380
+
381
+ class TestKhalibClassifier:
382
+ @pytest.mark.filterwarnings("ignore:'force_all_finite' was renamed")
383
+ # Notes:
384
+ # - SVC does not have predict_proba
385
+ # - MC LR OVR+normalization worsens the calibration
386
+ @pytest.mark.parametrize(
387
+ ("clf", "class_mode", "expected_ece"),
388
+ [
389
+ (KhiopsClassifier(), "binary", 0.005826),
390
+ (KhiopsClassifier(), "multi_class", 0.017015),
391
+ (LogisticRegression(), "binary", 0.009168),
392
+ (LogisticRegression(), "multi_class", 0.103714),
393
+ (SVC(), "binary", 0.006882),
394
+ (SVC(), "multi_class", 0.052011),
395
+ (None, "binary", 0.005826),
396
+ (None, "multi_class", 0.017015),
397
+ ],
398
+ )
399
+ @pytest.mark.parametrize(
400
+ ("train_mode", "train_size"),
401
+ [pytest.param("slow", 10000, marks=pytest.mark.slow), ("fast", 1000)],
402
+ )
403
+ @pytest.mark.parametrize("train_clf", [False, True])
404
+ def test_calibrator(
405
+ self, train_clf, train_mode, train_size, clf, class_mode, expected_ece, clf_data
406
+ ):
407
+ # Explose the input data to variables
408
+ x_train, x_calib, x_test, y_train, y_calib, y_test = clf_data[class_mode][
409
+ train_mode
410
+ ]
411
+
412
+ # Create the calibrated classifier
413
+ train_calib_split_state = 2
414
+ if clf is not None and train_clf:
415
+ # Control the train/calib split: This is only for reproducibility of the
416
+ # case where the classifier is fitted or None versos the pre-fitted case
417
+ x_not_test = np.vstack([x_train, x_calib])
418
+ y_not_test = np.hstack([y_train, y_calib])
419
+ x_train, x_calib, y_train, y_calib = train_test_split(
420
+ x_not_test,
421
+ y_not_test,
422
+ test_size=x_train.shape[0],
423
+ random_state=train_calib_split_state,
424
+ )
425
+
426
+ # Fit the classifier and calibrator
427
+ clf.fit(x_train, y_train)
428
+ cclf = KhalibClassifier(clf)
429
+ cclf.fit(x_calib, y_calib)
430
+ else:
431
+ cclf = KhalibClassifier(
432
+ clf, train_size=train_size, random_state=train_calib_split_state
433
+ )
434
+ cclf.fit(np.vstack([x_train, x_calib]), np.hstack([y_train, y_calib]))
435
+ check_is_fitted(cclf)
436
+
437
+ # We check the ECE only in "slow" mode because the train data is large enough
438
+ # The "fast" mode is only to test the correct execution
439
+ if train_mode == "slow":
440
+ # Check the resulting ECE is in the nominal range
441
+ y_scores_calib_test = cclf.predict_proba(x_test)
442
+ ece = calibration_error(
443
+ y_scores_calib_test, y_test, multi_class_method="top-label"
444
+ )
445
+ assert ece == pytest.approx(expected_ece, rel=1e-2)