fast-ivap 1.0.0__tar.gz

fast_ivap-1.0.0/.gitignore

@@ -0,0 +1,18 @@
+# Build artefacts
+build/
+dist/
+wheelhouse/
+*.so
+*.pyd
+
+# Python
+__pycache__/
+*.egg-info/
+.venv/
+.pytest_cache/
+.ruff_cache/
+
+# Editors / notebooks
+.vscode/
+.idea/
+.ipynb_checkpoints/

fast_ivap-1.0.0/CHANGELOG.md

@@ -0,0 +1,35 @@
+# Changelog
+
+## 1.0.0 - 2026-10-27
+
+Complete rewrite, published under the name `fast-ivap` (import `fast_ivap`).
+
+### Fixed
+- `p0` and `p1` were wrong for a test score equal to one of the calibration
+  scores.
+- `p1` was wrong for a test score above all calibration scores (it must be 1).
+- The pure-Python implementation failed on NumPy 2 (`np.cross` on 2-D vectors).
+- Removed the module-level global state, so separate predictors no longer
+  interfere with each other.
+
+### Changed
+- New API: `fast_ivap.IVAP` (`fit`, `predict_p0_p1`, `predict_proba`) and
+  `fast_ivap.sklearn.VennAbersCalibrator`, whose `predict_proba` now returns
+  an `(n, 2)` array as scikit-learn expects. `ScoresToMultiProbs` is kept as a
+  deprecated shim.
+- The C++ code is now a dependency-free header-only library, with nanobind
+  bindings (zero-copy NumPy input, GIL released) built by scikit-build-core.
+- Portable wheels for Linux, macOS and Windows via cibuildwheel.
+
+### Added
+- `merge_log_minimax` (IVAP and CVAP) and `merge_brier`.
+- pytest suite checking both backends against IVAP computed from its
+  definition; C++ tests via CTest; CI on Linux, macOS and Windows.
+
+## 0.2 - 2020-07
+- Fixed bug affecting the `p0` calculation; added a test notebook.
+- (2024, branches `cpp_version` / `pip_package`) C++ port with pybind11 and a
+  scikit-learn style wrapper.
+
+## 0.1
+- Initial pure-Python implementation.

fast_ivap-1.0.0/CMakeLists.txt

@@ -0,0 +1,48 @@
+cmake_minimum_required(VERSION 3.18...3.31)
+project(fast_ivap LANGUAGES CXX)
+
+# Header-only C++ library: fast_ivap::fast_ivap
+add_library(fast_ivap INTERFACE)
+add_library(fast_ivap::fast_ivap ALIAS fast_ivap)
+target_include_directories(
+  fast_ivap INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+                      $<INSTALL_INTERFACE:include>)
+target_compile_features(fast_ivap INTERFACE cxx_std_20)
+
+if(SKBUILD)
+  # Python extension module fast_ivap._core, built by scikit-build-core.
+  find_package(Python 3.11 REQUIRED
+               COMPONENTS Interpreter Development.Module ${SKBUILD_SABI_COMPONENT})
+  find_package(nanobind CONFIG REQUIRED)
+  nanobind_add_module(_core STABLE_ABI NB_STATIC src/bindings/core.cpp)
+  target_link_libraries(_core PRIVATE fast_ivap)
+  install(TARGETS _core LIBRARY DESTINATION fast_ivap)
+  return()
+endif()
+
+# Standalone C++ build: tests, example and install rules for find_package().
+option(FAST_IVAP_BUILD_TESTS "Build the C++ tests" ${PROJECT_IS_TOP_LEVEL})
+option(FAST_IVAP_BUILD_EXAMPLES "Build the C++ example" ${PROJECT_IS_TOP_LEVEL})
+
+if(FAST_IVAP_BUILD_TESTS)
+  enable_testing()
+  add_executable(test_ivap tests/cpp/test_ivap.cpp)
+  target_link_libraries(test_ivap PRIVATE fast_ivap)
+  add_test(NAME test_ivap COMMAND test_ivap)
+endif()
+
+if(FAST_IVAP_BUILD_EXAMPLES)
+  add_executable(ivap_example examples/cpp/example.cpp)
+  target_link_libraries(ivap_example PRIVATE fast_ivap)
+endif()
+
+include(GNUInstallDirs)
+include(CMakePackageConfigHelpers)
+install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+install(TARGETS fast_ivap EXPORT fast_ivapTargets)
+install(EXPORT fast_ivapTargets NAMESPACE fast_ivap::
+        DESTINATION ${CMAKE_INSTALL_DATADIR}/cmake/fast_ivap)
+file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/fast_ivapConfig.cmake
+     "include(\"\${CMAKE_CURRENT_LIST_DIR}/fast_ivapTargets.cmake\")\n")
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/fast_ivapConfig.cmake
+        DESTINATION ${CMAKE_INSTALL_DATADIR}/cmake/fast_ivap)

fast_ivap-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Paolo Toccaceli, Royal Holloway, Univ. of London
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

fast_ivap-1.0.0/PKG-INFO

@@ -0,0 +1,188 @@
+Metadata-Version: 2.4
+Name: fast-ivap
+Version: 1.0.0
+Summary: Fast inductive Venn-ABERS predictors: distribution-free calibration of classifier scores
+Keywords: calibration,conformal prediction,venn-abers,isotonic regression,probability
+Author: Paolo Toccaceli
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Project-URL: Homepage, https://github.com/ptocca/VennABERS
+Project-URL: Paper, https://arxiv.org/abs/1511.00213
+Requires-Python: >=3.11
+Requires-Dist: numpy>=1.25
+Provides-Extra: sklearn
+Requires-Dist: scikit-learn>=1.6; extra == "sklearn"
+Description-Content-Type: text/markdown
+
+# fast-ivap: fast inductive Venn-ABERS predictors
+
+[![CI](https://github.com/ptocca/VennABERS/actions/workflows/ci.yml/badge.svg)](https://github.com/ptocca/VennABERS/actions/workflows/ci.yml)
+
+`fast-ivap` turns the scores of any binary classifier into **well-calibrated
+probabilities**, with no assumption beyond the calibration and test data being
+i.i.d. It implements the inductive Venn-ABERS predictor (IVAP) with the
+O(k log k) algorithm of
+
+> V. Vovk, I. Petej, V. Fedorova. *Large-scale probabilistic predictors with
+> and without guarantees of validity.* NeurIPS 2015.
+> [arXiv:1511.00213](https://arxiv.org/abs/1511.00213)
+
+The core is a small header-only C++20 library with Python bindings
+([nanobind](https://github.com/wjakob/nanobind)), plus a pure-Python
+reference implementation. A gentle introduction to Venn-ABERS predictors is in
+this [tutorial](https://cml.rhul.ac.uk/people/ptocca/HomePage/Toccaceli_CP___Venn_Tutorial.pdf).
+
+## What it computes
+
+Given calibration scores and labels, and the score `s` of a test object, an
+IVAP outputs two probabilities that the label is 1:
+
+- `p0`: the isotonic-regression fit at `s` after adding `(s, 0)` to the
+  calibration set,
+- `p1`: the same after adding `(s, 1)`.
+
+`p0 <= p1` always holds, and one of the two (which one depends on the unknown
+true label) is perfectly calibrated. The gap `p1 - p0` is typically small and
+shrinks as the calibration set grows. A single probability can be
+obtained with the log-loss minimax merge
+
+$$p = \frac{p_1}{1 - p_0 + p_1}.$$
+
+Fitting takes O(k log k) for k calibration examples; each prediction takes
+O(log k).
+
+## Installation
+
+```bash
+pip install fast-ivap              # NumPy API
+pip install "fast-ivap[sklearn]"   # plus the scikit-learn estimator
+```
+
+Until the package is on PyPI, install from GitHub (this builds from source, so
+it needs a C++20 compiler; CMake is fetched automatically if missing):
+
+```bash
+pip install git+https://github.com/ptocca/VennABERS
+```
+
+Prebuilt wheels are produced for Linux (x86-64, aarch64; glibc and musl),
+macOS (Intel and Apple silicon) and Windows (x86-64, ARM64), for CPython 3.11
+and a single stable-ABI wheel for CPython 3.12 and later.
+
+## Usage
+
+### NumPy
+
+```python
+from fast_ivap import IVAP
+
+ivap = IVAP().fit(cal_scores, cal_labels)  # labels are 0/1 or booleans
+p0, p1 = ivap.predict_p0_p1(test_scores)  # Venn-ABERS pair
+p = ivap.predict_proba(test_scores)  # merged probability of label 1
+```
+
+See [`examples/quickstart.py`](examples/quickstart.py) for a complete example
+with a scikit-learn classifier.
+
+For a cross Venn-ABERS predictor (CVAP), fit one IVAP per fold, stack the
+outputs and merge them with geometric means:
+
+```python
+from fast_ivap import merge_log_minimax
+
+p = merge_log_minimax(p0_folds, p1_folds, axis=0)  # arrays of shape (K, n)
+```
+
+`merge_brier` gives the Brier-loss minimax merge instead.
+
+### scikit-learn
+
+`VennAbersCalibrator` takes one score per example and follows the
+scikit-learn classifier API (`predict_proba` returns `[1 - p, p]`):
+
+```python
+from fast_ivap.sklearn import VennAbersCalibrator
+
+cal = VennAbersCalibrator().fit(clf.decision_function(X_cal), y_cal)
+proba = cal.predict_proba(clf.decision_function(X_test))
+p0, p1 = cal.predict_p0_p1(clf.decision_function(X_test))
+```
+
+### C++
+
+The C++ library is the single header
+[`include/fast_ivap/ivap.hpp`](include/fast_ivap/ivap.hpp) with no
+dependencies. Copy it into your project, or install it and use CMake:
+
+```bash
+cmake -S . -B build && cmake --install build --prefix /your/prefix
+```
+
+```cmake
+find_package(fast_ivap REQUIRED)
+target_link_libraries(my_app PRIVATE fast_ivap::fast_ivap)
+```
+
+```cpp
+#include <fast_ivap/ivap.hpp>
+
+auto cal = fast_ivap::fit(scores, labels);           // any contiguous range of doubles
+auto [p0, p1] = fast_ivap::predict(cal, test_scores);
+```
+
+## Implementation notes
+
+- The C++ backend is used when available. `IVAP(backend="python")` or the
+  environment variable `FAST_IVAP_BACKEND=python` select the pure-Python
+  implementation in [`_reference.py`](src/fast_ivap/_reference.py), which
+  follows the paper step by step. Both are tested against each other and
+  against a brute-force IVAP computed from its definition.
+- `p1` is computed with Algorithms 1 and 2 of the paper. Instead of the
+  mirror-image Algorithms 3 and 4, `p0` is obtained from the same code by the
+  symmetry noted in the paper: negate the scores, flip the labels, and take
+  `1 - p1`.
+- The paper defines the extra CSD point for Algorithms 3–4 as
+  `P[k'+1] = P[k'] + (1, 1)`; it should be `P[k'] + (1, 0)`, because that
+  segment represents the test object with label 0. The original
+  implementation already used `(1, 0)`.
+- A test score equal to a calibration score, and a test score outside the
+  range of the calibration scores, are handled exactly as in the definition
+  (two bugs in versions up to 0.2 affected these cases).
+- A NaN test score yields NaN probabilities. Calibration scores must be finite.
+
+## Development
+
+```bash
+uv sync --group dev      # build the extension and install dev tools
+uv run pytest            # Python tests (both backends)
+uv run ruff check . && uv run ruff format --check .
+
+cmake -S . -B build && cmake --build build && ctest --test-dir build   # C++ tests
+```
+
+The GitHub Actions workflows run the tests on Linux, macOS and Windows, and
+build wheels with [cibuildwheel](https://cibuildwheel.pypa.io/). Publishing a
+GitHub release uploads the wheels and sdist to PyPI (trusted publishing).
+
+## Version history
+
+See [CHANGELOG.md](CHANGELOG.md).
+
+## Acknowledgements
+
+- The algorithm was invented by Prof. Vladimir Vovk et al. at the
+  [Centre for Reliable Machine Learning, Royal Holloway, University of London](https://cml.rhul.ac.uk/).
+- This work was supported in part by funding from the European Union's Horizon
+  2020 Research and Innovation programme under Grant Agreement no. 671555
+  (ExCAPE).
+- We are grateful to the Ministry of Education, Youth and Sports (Czech
+  Republic) that supports the Large Infrastructures for Research, Experimental
+  Development and Innovations project "IT4Innovations National Supercomputing
+  Center - LM2015070" for providing access to the Solomon supercomputing
+  cluster.

fast_ivap-1.0.0/README.md

@@ -0,0 +1,166 @@
+# fast-ivap: fast inductive Venn-ABERS predictors
+
+[![CI](https://github.com/ptocca/VennABERS/actions/workflows/ci.yml/badge.svg)](https://github.com/ptocca/VennABERS/actions/workflows/ci.yml)
+
+`fast-ivap` turns the scores of any binary classifier into **well-calibrated
+probabilities**, with no assumption beyond the calibration and test data being
+i.i.d. It implements the inductive Venn-ABERS predictor (IVAP) with the
+O(k log k) algorithm of
+
+> V. Vovk, I. Petej, V. Fedorova. *Large-scale probabilistic predictors with
+> and without guarantees of validity.* NeurIPS 2015.
+> [arXiv:1511.00213](https://arxiv.org/abs/1511.00213)
+
+The core is a small header-only C++20 library with Python bindings
+([nanobind](https://github.com/wjakob/nanobind)), plus a pure-Python
+reference implementation. A gentle introduction to Venn-ABERS predictors is in
+this [tutorial](https://cml.rhul.ac.uk/people/ptocca/HomePage/Toccaceli_CP___Venn_Tutorial.pdf).
+
+## What it computes
+
+Given calibration scores and labels, and the score `s` of a test object, an
+IVAP outputs two probabilities that the label is 1:
+
+- `p0`: the isotonic-regression fit at `s` after adding `(s, 0)` to the
+  calibration set,
+- `p1`: the same after adding `(s, 1)`.
+
+`p0 <= p1` always holds, and one of the two (which one depends on the unknown
+true label) is perfectly calibrated. The gap `p1 - p0` is typically small and
+shrinks as the calibration set grows. A single probability can be
+obtained with the log-loss minimax merge
+
+$$p = \frac{p_1}{1 - p_0 + p_1}.$$
+
+Fitting takes O(k log k) for k calibration examples; each prediction takes
+O(log k).
+
+## Installation
+
+```bash
+pip install fast-ivap              # NumPy API
+pip install "fast-ivap[sklearn]"   # plus the scikit-learn estimator
+```
+
+Until the package is on PyPI, install from GitHub (this builds from source, so
+it needs a C++20 compiler; CMake is fetched automatically if missing):
+
+```bash
+pip install git+https://github.com/ptocca/VennABERS
+```
+
+Prebuilt wheels are produced for Linux (x86-64, aarch64; glibc and musl),
+macOS (Intel and Apple silicon) and Windows (x86-64, ARM64), for CPython 3.11
+and a single stable-ABI wheel for CPython 3.12 and later.
+
+## Usage
+
+### NumPy
+
+```python
+from fast_ivap import IVAP
+
+ivap = IVAP().fit(cal_scores, cal_labels)  # labels are 0/1 or booleans
+p0, p1 = ivap.predict_p0_p1(test_scores)  # Venn-ABERS pair
+p = ivap.predict_proba(test_scores)  # merged probability of label 1
+```
+
+See [`examples/quickstart.py`](examples/quickstart.py) for a complete example
+with a scikit-learn classifier.
+
+For a cross Venn-ABERS predictor (CVAP), fit one IVAP per fold, stack the
+outputs and merge them with geometric means:
+
+```python
+from fast_ivap import merge_log_minimax
+
+p = merge_log_minimax(p0_folds, p1_folds, axis=0)  # arrays of shape (K, n)
+```
+
+`merge_brier` gives the Brier-loss minimax merge instead.
+
+### scikit-learn
+
+`VennAbersCalibrator` takes one score per example and follows the
+scikit-learn classifier API (`predict_proba` returns `[1 - p, p]`):
+
+```python
+from fast_ivap.sklearn import VennAbersCalibrator
+
+cal = VennAbersCalibrator().fit(clf.decision_function(X_cal), y_cal)
+proba = cal.predict_proba(clf.decision_function(X_test))
+p0, p1 = cal.predict_p0_p1(clf.decision_function(X_test))
+```
+
+### C++
+
+The C++ library is the single header
+[`include/fast_ivap/ivap.hpp`](include/fast_ivap/ivap.hpp) with no
+dependencies. Copy it into your project, or install it and use CMake:
+
+```bash
+cmake -S . -B build && cmake --install build --prefix /your/prefix
+```
+
+```cmake
+find_package(fast_ivap REQUIRED)
+target_link_libraries(my_app PRIVATE fast_ivap::fast_ivap)
+```
+
+```cpp
+#include <fast_ivap/ivap.hpp>
+
+auto cal = fast_ivap::fit(scores, labels);           // any contiguous range of doubles
+auto [p0, p1] = fast_ivap::predict(cal, test_scores);
+```
+
+## Implementation notes
+
+- The C++ backend is used when available. `IVAP(backend="python")` or the
+  environment variable `FAST_IVAP_BACKEND=python` select the pure-Python
+  implementation in [`_reference.py`](src/fast_ivap/_reference.py), which
+  follows the paper step by step. Both are tested against each other and
+  against a brute-force IVAP computed from its definition.
+- `p1` is computed with Algorithms 1 and 2 of the paper. Instead of the
+  mirror-image Algorithms 3 and 4, `p0` is obtained from the same code by the
+  symmetry noted in the paper: negate the scores, flip the labels, and take
+  `1 - p1`.
+- The paper defines the extra CSD point for Algorithms 3–4 as
+  `P[k'+1] = P[k'] + (1, 1)`; it should be `P[k'] + (1, 0)`, because that
+  segment represents the test object with label 0. The original
+  implementation already used `(1, 0)`.
+- A test score equal to a calibration score, and a test score outside the
+  range of the calibration scores, are handled exactly as in the definition
+  (two bugs in versions up to 0.2 affected these cases).
+- A NaN test score yields NaN probabilities. Calibration scores must be finite.
+
+## Development
+
+```bash
+uv sync --group dev      # build the extension and install dev tools
+uv run pytest            # Python tests (both backends)
+uv run ruff check . && uv run ruff format --check .
+
+cmake -S . -B build && cmake --build build && ctest --test-dir build   # C++ tests
+```
+
+The GitHub Actions workflows run the tests on Linux, macOS and Windows, and
+build wheels with [cibuildwheel](https://cibuildwheel.pypa.io/). Publishing a
+GitHub release uploads the wheels and sdist to PyPI (trusted publishing).
+
+## Version history
+
+See [CHANGELOG.md](CHANGELOG.md).
+
+## Acknowledgements
+
+- The algorithm was invented by Prof. Vladimir Vovk et al. at the
+  [Centre for Reliable Machine Learning, Royal Holloway, University of London](https://cml.rhul.ac.uk/).
+- This work was supported in part by funding from the European Union's Horizon
+  2020 Research and Innovation programme under Grant Agreement no. 671555
+  (ExCAPE).
+- We are grateful to the Ministry of Education, Youth and Sports (Czech
+  Republic) that supports the Large Infrastructures for Research, Experimental
+  Development and Innovations project "IT4Innovations National Supercomputing
+  Center - LM2015070" for providing access to the Solomon supercomputing
+  cluster.

fast_ivap-1.0.0/examples/cpp/example.cpp

@@ -0,0 +1,21 @@
+// Minimal example of using the header-only C++ library.
+//
+//   cmake -B build && cmake --build build && ./build/ivap_example
+
+#include <fast_ivap/ivap.hpp>
+
+#include <cstdio>
+#include <vector>
+
+int main() {
+    const std::vector<double> scores{2.0, 3.0, 4.0, 5.0, 7.0, 8.0};
+    const std::vector<double> labels{0.0, 1.0, 0.0, 0.0, 1.0, 1.0};
+    const std::vector<double> test{1.0, 2.5, 7.2};
+
+    const fast_ivap::Calibrator cal = fast_ivap::fit(scores, labels);
+    const auto [p0, p1] = fast_ivap::predict(cal, test);
+
+    for (std::size_t i = 0; i < test.size(); ++i) {
+        std::printf("score %4.1f  ->  p0 = %.3f, p1 = %.3f\n", test[i], p0[i], p1[i]);
+    }
+}

fast_ivap-1.0.0/examples/quickstart.py

@@ -0,0 +1,34 @@
+"""Calibrate a classifier's scores with an inductive Venn-ABERS predictor.
+
+Run with:  python examples/quickstart.py   (needs scikit-learn)
+"""
+
+import numpy as np
+from sklearn.datasets import make_classification
+from sklearn.model_selection import train_test_split
+from sklearn.svm import LinearSVC
+
+from fast_ivap import IVAP
+
+X, y = make_classification(n_samples=20_000, n_features=20, flip_y=0.1, random_state=0)
+X_train, X_rest, y_train, y_rest = train_test_split(X, y, train_size=0.5, random_state=0)
+X_cal, X_test, y_cal, y_test = train_test_split(X_rest, y_rest, train_size=0.5, random_state=0)
+
+# 1. Train any scoring classifier on the proper training set.
+clf = LinearSVC().fit(X_train, y_train)
+
+# 2. Fit the IVAP on the classifier's scores for a held-out calibration set.
+ivap = IVAP().fit(clf.decision_function(X_cal), y_cal)
+
+# 3. Predict: a pair (p0, p1) per test example, or a single merged probability.
+test_scores = clf.decision_function(X_test)
+p0, p1 = ivap.predict_p0_p1(test_scores)
+p = ivap.predict_proba(test_scores)
+
+print("first five test examples:")
+for s, a, b, q, label in zip(test_scores[:5], p0, p1, p, y_test, strict=False):
+    print(f"  score {s:+.2f}  p0 {a:.3f}  p1 {b:.3f}  p {q:.3f}  true label {label}")
+
+print(f"\nmean width p1 - p0:   {np.mean(p1 - p0):.4f}")
+print(f"Brier score:          {np.mean((p - y_test) ** 2):.4f}")
+print(f"mean predicted prob.: {p.mean():.4f}   observed frequency: {y_test.mean():.4f}")