calibroute-ai 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calibroute_ai-0.3.0/.github/workflows/ci.yml +24 -0
- calibroute_ai-0.3.0/.github/workflows/publish.yml +47 -0
- calibroute_ai-0.3.0/.gitignore +13 -0
- calibroute_ai-0.3.0/CHANGELOG.md +29 -0
- calibroute_ai-0.3.0/CITATION.cff +11 -0
- calibroute_ai-0.3.0/CONTRIBUTING.md +23 -0
- calibroute_ai-0.3.0/LICENSE +21 -0
- calibroute_ai-0.3.0/PKG-INFO +168 -0
- calibroute_ai-0.3.0/README.md +121 -0
- calibroute_ai-0.3.0/ROADMAP.md +45 -0
- calibroute_ai-0.3.0/SECURITY.md +12 -0
- calibroute_ai-0.3.0/docs/design.md +80 -0
- calibroute_ai-0.3.0/docs/releasing.md +50 -0
- calibroute_ai-0.3.0/examples/financial_ner/PROVENANCE.md +32 -0
- calibroute_ai-0.3.0/examples/financial_ner/README.md +71 -0
- calibroute_ai-0.3.0/examples/financial_ner/encoder_audit.json +196 -0
- calibroute_ai-0.3.0/examples/financial_ner/encoder_audit.md +34 -0
- calibroute_ai-0.3.0/examples/financial_ner/encoder_policy.json +30 -0
- calibroute_ai-0.3.0/examples/financial_ner/encoder_seed42.csv +1050 -0
- calibroute_ai-0.3.0/examples/financial_ner/generative_audit.json +196 -0
- calibroute_ai-0.3.0/examples/financial_ner/generative_audit.md +34 -0
- calibroute_ai-0.3.0/examples/financial_ner/generative_self_consistency_seed42.csv +1050 -0
- calibroute_ai-0.3.0/examples/financial_ner/routing_summary.json +89 -0
- calibroute_ai-0.3.0/examples/financial_ner/run_case_study.py +39 -0
- calibroute_ai-0.3.0/examples/production_batch.csv +7 -0
- calibroute_ai-0.3.0/examples/validation.csv +22 -0
- calibroute_ai-0.3.0/pyproject.toml +52 -0
- calibroute_ai-0.3.0/src/calibroute/__init__.py +25 -0
- calibroute_ai-0.3.0/src/calibroute/adapters/__init__.py +5 -0
- calibroute_ai-0.3.0/src/calibroute/adapters/financial_ner.py +81 -0
- calibroute_ai-0.3.0/src/calibroute/bounds.py +50 -0
- calibroute_ai-0.3.0/src/calibroute/cli.py +135 -0
- calibroute_ai-0.3.0/src/calibroute/io.py +72 -0
- calibroute_ai-0.3.0/src/calibroute/metrics.py +145 -0
- calibroute_ai-0.3.0/src/calibroute/models.py +58 -0
- calibroute_ai-0.3.0/src/calibroute/policy.py +267 -0
- calibroute_ai-0.3.0/src/calibroute/report.py +63 -0
- calibroute_ai-0.3.0/src/calibroute/shift.py +82 -0
- calibroute_ai-0.3.0/src/calibroute/validation.py +46 -0
- calibroute_ai-0.3.0/tests/test_adapter.py +64 -0
- calibroute_ai-0.3.0/tests/test_bounds.py +23 -0
- calibroute_ai-0.3.0/tests/test_io.py +36 -0
- calibroute_ai-0.3.0/tests/test_metrics.py +50 -0
- calibroute_ai-0.3.0/tests/test_policy.py +124 -0
- calibroute_ai-0.3.0/tests/test_regressions.py +114 -0
- calibroute_ai-0.3.0/tests/test_shift.py +34 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
name: ci
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
test:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
strategy:
|
|
11
|
+
matrix:
|
|
12
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v4
|
|
15
|
+
- uses: actions/setup-python@v5
|
|
16
|
+
with:
|
|
17
|
+
python-version: ${{ matrix.python-version }}
|
|
18
|
+
- run: python -m pip install -e ".[dev]"
|
|
19
|
+
- run: ruff check src tests examples/financial_ner/run_case_study.py
|
|
20
|
+
- run: ruff format --check src tests examples/financial_ner/run_case_study.py
|
|
21
|
+
- run: python -m unittest discover -s tests -v
|
|
22
|
+
- run: python examples/financial_ner/run_case_study.py
|
|
23
|
+
- run: python -m build
|
|
24
|
+
- run: python -m twine check dist/*
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
|
|
6
|
+
permissions:
|
|
7
|
+
contents: read
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: pypi-publication
|
|
11
|
+
cancel-in-progress: false
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
if: github.ref == 'refs/heads/main'
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v4
|
|
19
|
+
- uses: actions/setup-python@v5
|
|
20
|
+
with:
|
|
21
|
+
python-version: "3.12"
|
|
22
|
+
- run: python -m pip install -e ".[dev]"
|
|
23
|
+
- run: python -m unittest discover -s tests -v
|
|
24
|
+
- run: ruff check src tests examples/financial_ner/run_case_study.py
|
|
25
|
+
- run: python -m build
|
|
26
|
+
- run: python -m twine check dist/*
|
|
27
|
+
- uses: actions/upload-artifact@v4
|
|
28
|
+
with:
|
|
29
|
+
name: pypi-distributions
|
|
30
|
+
path: dist/
|
|
31
|
+
if-no-files-found: error
|
|
32
|
+
|
|
33
|
+
publish:
|
|
34
|
+
needs: build
|
|
35
|
+
runs-on: ubuntu-latest
|
|
36
|
+
environment:
|
|
37
|
+
name: pypi
|
|
38
|
+
url: https://pypi.org/project/calibroute-ai/
|
|
39
|
+
permissions:
|
|
40
|
+
id-token: write
|
|
41
|
+
steps:
|
|
42
|
+
- uses: actions/download-artifact@v4
|
|
43
|
+
with:
|
|
44
|
+
name: pypi-distributions
|
|
45
|
+
path: dist/
|
|
46
|
+
- name: Publish verified distributions using Trusted Publishing
|
|
47
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.3.0 - 2026-09-20
|
|
4
|
+
|
|
5
|
+
- Fix tie groups in threshold fitting, risk-coverage curves, and selective reports.
|
|
6
|
+
- Add independent holdout validation with CLI pass/fail exit codes.
|
|
7
|
+
- Clarify pointwise fit bounds versus independent fixed-policy risk assessment.
|
|
8
|
+
- Replace quadratic AUROC with tie-aware O(n log n) rank aggregation.
|
|
9
|
+
- Make missing shift threshold default to adaptive monitoring; validate inputs.
|
|
10
|
+
- Regenerate case-study reports and record provenance and reproducibility limits.
|
|
11
|
+
- Align package/citation versions; expand CI to lint, build, and check distributions.
|
|
12
|
+
|
|
13
|
+
Refit 0.2 policies before deployment. The 0.2 policy selection and audit reports
|
|
14
|
+
could split ties and depend on input order. No 0.2 PyPI release is intended.
|
|
15
|
+
|
|
16
|
+
## 0.2.0 - 2026-09-19
|
|
17
|
+
|
|
18
|
+
- Add exact Clopper-Pearson upper risk bounds to policy fitting.
|
|
19
|
+
- Add batch-size-aware JSD calibration and explicit small-batch handling.
|
|
20
|
+
- Add a Financial NER sentence-output adapter and multi-domain case study.
|
|
21
|
+
- Expand policy schema to record declared risk and calibration settings.
|
|
22
|
+
|
|
23
|
+
## 0.1.0 - 2026-09-19
|
|
24
|
+
|
|
25
|
+
- Initial model-agnostic confidence audit.
|
|
26
|
+
- Validation-grounded selective policy fitting.
|
|
27
|
+
- Batch confidence-shift detection.
|
|
28
|
+
- Auditable accept, human-review, and abstain routing.
|
|
29
|
+
- Dependency-free Python API and CLI.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
cff-version: 1.2.0
|
|
2
|
+
message: "If CalibRoute supports your research, please cite the software release."
|
|
3
|
+
title: "CalibRoute: Uncertainty-Aware Evaluation and Decision Control for AI Systems"
|
|
4
|
+
type: software
|
|
5
|
+
version: 0.3.0
|
|
6
|
+
date-released: 2026-09-20
|
|
7
|
+
repository-code: "https://github.com/Garyouki/calibroute"
|
|
8
|
+
authors:
|
|
9
|
+
- family-names: "Zheng"
|
|
10
|
+
given-names: "Zihao"
|
|
11
|
+
license: MIT
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Contributions are welcome, especially new metrics, shift-monitor adapters,
|
|
4
|
+
task-specific examples, and tests that expose unsafe assumptions.
|
|
5
|
+
|
|
6
|
+
## Local workflow
|
|
7
|
+
|
|
8
|
+
1. Create a focused branch.
|
|
9
|
+
2. Install the package with `python -m pip install -e .`.
|
|
10
|
+
3. Run `python -m unittest discover -s tests -v`.
|
|
11
|
+
4. Add tests for every behavior change.
|
|
12
|
+
5. Document whether a new metric is descriptive, empirically calibrated, or
|
|
13
|
+
accompanied by a formal guarantee.
|
|
14
|
+
|
|
15
|
+
Please do not commit private, proprietary, personally identifiable, or
|
|
16
|
+
license-restricted datasets. Prefer download scripts, hashes, and synthetic
|
|
17
|
+
fixtures for examples.
|
|
18
|
+
|
|
19
|
+
## Pull requests
|
|
20
|
+
|
|
21
|
+
Keep pull requests narrow. Explain the uncertainty assumption, expected failure
|
|
22
|
+
mode, and how the change affects accept/review/abstain decisions.
|
|
23
|
+
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Zihao Zheng and CalibRoute contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: calibroute-ai
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Uncertainty-aware evaluation and decision control for AI systems.
|
|
5
|
+
Project-URL: Repository, https://github.com/Garyouki/calibroute
|
|
6
|
+
Project-URL: Issues, https://github.com/Garyouki/calibroute/issues
|
|
7
|
+
Project-URL: Documentation, https://github.com/Garyouki/calibroute/blob/main/README.md
|
|
8
|
+
Author: Zihao Zheng
|
|
9
|
+
License: MIT License
|
|
10
|
+
|
|
11
|
+
Copyright (c) 2026 Zihao Zheng and CalibRoute contributors
|
|
12
|
+
|
|
13
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
14
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
15
|
+
in the Software without restriction, including without limitation the rights
|
|
16
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
17
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
18
|
+
furnished to do so, subject to the following conditions:
|
|
19
|
+
|
|
20
|
+
The above copyright notice and this permission notice shall be included in all
|
|
21
|
+
copies or substantial portions of the Software.
|
|
22
|
+
|
|
23
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
24
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
25
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
26
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
27
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
28
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
29
|
+
SOFTWARE.
|
|
30
|
+
License-File: LICENSE
|
|
31
|
+
Keywords: abstention,ai-evaluation,distribution-shift,selective-prediction,trustworthy-ai,uncertainty
|
|
32
|
+
Classifier: Development Status :: 3 - Alpha
|
|
33
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
34
|
+
Classifier: Programming Language :: Python :: 3
|
|
35
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
36
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
37
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
38
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
39
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
40
|
+
Requires-Python: >=3.10
|
|
41
|
+
Provides-Extra: dev
|
|
42
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
43
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
44
|
+
Requires-Dist: ruff>=0.8; extra == 'dev'
|
|
45
|
+
Requires-Dist: twine>=6; extra == 'dev'
|
|
46
|
+
Description-Content-Type: text/markdown
|
|
47
|
+
|
|
48
|
+
# CalibRoute
|
|
49
|
+
|
|
50
|
+
CalibRoute is a lightweight Python toolkit for confidence evaluation and
|
|
51
|
+
uncertainty-aware routing. It converts model predictions into three actions:
|
|
52
|
+
`accept`, `human_review`, or `abstain`.
|
|
53
|
+
|
|
54
|
+
## Features
|
|
55
|
+
|
|
56
|
+
- Confidence calibration and error-ranking metrics
|
|
57
|
+
- Risk-coverage analysis
|
|
58
|
+
- Validation-only threshold fitting
|
|
59
|
+
- Tie-aware threshold fitting and independent holdout risk assessment
|
|
60
|
+
- Batch-size-aware confidence-shift detection
|
|
61
|
+
- Financial NER encoder and generative-output adapter
|
|
62
|
+
- CSV and JSONL support
|
|
63
|
+
- Dependency-free Python API and CLI
|
|
64
|
+
|
|
65
|
+
## Install
|
|
66
|
+
|
|
67
|
+
Requires Python 3.10 or newer.
|
|
68
|
+
|
|
69
|
+
Install with `python -m pip install calibroute-ai`.
|
|
70
|
+
|
|
71
|
+
To run the checked-in examples or contribute, clone the repository:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
git clone https://github.com/Garyouki/calibroute.git
|
|
75
|
+
cd calibroute
|
|
76
|
+
python -m pip install -e .
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
The distribution name is `calibroute-ai`; the Python import and command are
|
|
80
|
+
`calibroute`. No API key or model service is needed.
|
|
81
|
+
|
|
82
|
+
## Quick start
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
# Audit labeled predictions
|
|
86
|
+
calibroute audit \
|
|
87
|
+
--input examples/validation.csv \
|
|
88
|
+
--output examples/audit.md
|
|
89
|
+
|
|
90
|
+
# Fit an acceptance policy on validation data
|
|
91
|
+
calibroute fit \
|
|
92
|
+
--input examples/validation.csv \
|
|
93
|
+
--max-risk 0.20 \
|
|
94
|
+
--min-coverage 0.25 \
|
|
95
|
+
--risk-method empirical \
|
|
96
|
+
--output examples/policy.json
|
|
97
|
+
|
|
98
|
+
# Route a new prediction batch
|
|
99
|
+
calibroute route \
|
|
100
|
+
--input examples/production_batch.csv \
|
|
101
|
+
--policy examples/policy.json \
|
|
102
|
+
--output examples/decisions.csv \
|
|
103
|
+
--summary examples/routing-summary.json
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
## Input format
|
|
107
|
+
|
|
108
|
+
| Field | Required | Description |
|
|
109
|
+
|---|---|---|
|
|
110
|
+
| `id` | recommended | Prediction identifier |
|
|
111
|
+
| `confidence` | yes | Number between 0 and 1 |
|
|
112
|
+
| `correct` | `audit` and `fit` only | Boolean or `1`/`0` label |
|
|
113
|
+
| `domain` | no | Evaluation slice or deployment domain |
|
|
114
|
+
|
|
115
|
+
Additional columns are preserved as metadata.
|
|
116
|
+
|
|
117
|
+
The default fit searches thresholds using pointwise 95% Clopper-Pearson bounds.
|
|
118
|
+
Threshold selection on the same labels does **not** provide a 95% guarantee for
|
|
119
|
+
the selected policy. Freeze the policy, then assess it on a separate IID holdout:
|
|
120
|
+
|
|
121
|
+
```bash
|
|
122
|
+
calibroute validate --input holdout.csv --policy examples/policy.json --output holdout-report.json
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
Exit code 0 means the holdout upper bound meets the declared risk limit; 1 means
|
|
126
|
+
it does not (including zero accepted samples); 2 means invalid input. Never tune
|
|
127
|
+
against this holdout or reuse it to select among multiple policies. A pass does
|
|
128
|
+
not cover distribution shift. See [statistical scope](docs/design.md).
|
|
129
|
+
|
|
130
|
+
## Python API
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
from calibroute import PredictionRecord, fit_policy, route_batch
|
|
134
|
+
|
|
135
|
+
validation = [
|
|
136
|
+
PredictionRecord("a", 0.98, True),
|
|
137
|
+
PredictionRecord("b", 0.82, True),
|
|
138
|
+
PredictionRecord("c", 0.55, False),
|
|
139
|
+
]
|
|
140
|
+
|
|
141
|
+
policy = fit_policy(validation, max_risk=0.10, min_coverage=0.50,
|
|
142
|
+
risk_method="empirical") # Tiny illustrative sample only.
|
|
143
|
+
decisions, summary = route_batch(
|
|
144
|
+
[PredictionRecord("new", 0.74)],
|
|
145
|
+
policy,
|
|
146
|
+
)
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
## Limitations
|
|
150
|
+
|
|
151
|
+
CalibRoute is an evaluation and routing tool, not a safety certification.
|
|
152
|
+
Thresholds should be revalidated after changes to the model, task, prompt, or
|
|
153
|
+
deployment distribution. See [design principles](docs/design.md) for details.
|
|
154
|
+
|
|
155
|
+
## Development
|
|
156
|
+
|
|
157
|
+
```bash
|
|
158
|
+
python -m unittest discover -s tests -v
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
|
|
162
|
+
Release preparation is documented in [releasing](docs/releasing.md).
|
|
163
|
+
The [Financial NER case study](examples/financial_ner/README.md) demonstrates
|
|
164
|
+
the adapter and cross-domain failure pattern on 2,098 derived prediction rows.
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# CalibRoute
|
|
2
|
+
|
|
3
|
+
CalibRoute is a lightweight Python toolkit for confidence evaluation and
|
|
4
|
+
uncertainty-aware routing. It converts model predictions into three actions:
|
|
5
|
+
`accept`, `human_review`, or `abstain`.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- Confidence calibration and error-ranking metrics
|
|
10
|
+
- Risk-coverage analysis
|
|
11
|
+
- Validation-only threshold fitting
|
|
12
|
+
- Tie-aware threshold fitting and independent holdout risk assessment
|
|
13
|
+
- Batch-size-aware confidence-shift detection
|
|
14
|
+
- Financial NER encoder and generative-output adapter
|
|
15
|
+
- CSV and JSONL support
|
|
16
|
+
- Dependency-free Python API and CLI
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
Requires Python 3.10 or newer.
|
|
21
|
+
|
|
22
|
+
Install with `python -m pip install calibroute-ai`.
|
|
23
|
+
|
|
24
|
+
To run the checked-in examples or contribute, clone the repository:
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
git clone https://github.com/Garyouki/calibroute.git
|
|
28
|
+
cd calibroute
|
|
29
|
+
python -m pip install -e .
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The distribution name is `calibroute-ai`; the Python import and command are
|
|
33
|
+
`calibroute`. No API key or model service is needed.
|
|
34
|
+
|
|
35
|
+
## Quick start
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
# Audit labeled predictions
|
|
39
|
+
calibroute audit \
|
|
40
|
+
--input examples/validation.csv \
|
|
41
|
+
--output examples/audit.md
|
|
42
|
+
|
|
43
|
+
# Fit an acceptance policy on validation data
|
|
44
|
+
calibroute fit \
|
|
45
|
+
--input examples/validation.csv \
|
|
46
|
+
--max-risk 0.20 \
|
|
47
|
+
--min-coverage 0.25 \
|
|
48
|
+
--risk-method empirical \
|
|
49
|
+
--output examples/policy.json
|
|
50
|
+
|
|
51
|
+
# Route a new prediction batch
|
|
52
|
+
calibroute route \
|
|
53
|
+
--input examples/production_batch.csv \
|
|
54
|
+
--policy examples/policy.json \
|
|
55
|
+
--output examples/decisions.csv \
|
|
56
|
+
--summary examples/routing-summary.json
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Input format
|
|
60
|
+
|
|
61
|
+
| Field | Required | Description |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| `id` | recommended | Prediction identifier |
|
|
64
|
+
| `confidence` | yes | Number between 0 and 1 |
|
|
65
|
+
| `correct` | `audit` and `fit` only | Boolean or `1`/`0` label |
|
|
66
|
+
| `domain` | no | Evaluation slice or deployment domain |
|
|
67
|
+
|
|
68
|
+
Additional columns are preserved as metadata.
|
|
69
|
+
|
|
70
|
+
The default fit searches thresholds using pointwise 95% Clopper-Pearson bounds.
|
|
71
|
+
Threshold selection on the same labels does **not** provide a 95% guarantee for
|
|
72
|
+
the selected policy. Freeze the policy, then assess it on a separate IID holdout:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
calibroute validate --input holdout.csv --policy examples/policy.json --output holdout-report.json
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Exit code 0 means the holdout upper bound meets the declared risk limit; 1 means
|
|
79
|
+
it does not (including zero accepted samples); 2 means invalid input. Never tune
|
|
80
|
+
against this holdout or reuse it to select among multiple policies. A pass does
|
|
81
|
+
not cover distribution shift. See [statistical scope](docs/design.md).
|
|
82
|
+
|
|
83
|
+
## Python API
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
from calibroute import PredictionRecord, fit_policy, route_batch
|
|
87
|
+
|
|
88
|
+
validation = [
|
|
89
|
+
PredictionRecord("a", 0.98, True),
|
|
90
|
+
PredictionRecord("b", 0.82, True),
|
|
91
|
+
PredictionRecord("c", 0.55, False),
|
|
92
|
+
]
|
|
93
|
+
|
|
94
|
+
policy = fit_policy(validation, max_risk=0.10, min_coverage=0.50,
|
|
95
|
+
risk_method="empirical") # Tiny illustrative sample only.
|
|
96
|
+
decisions, summary = route_batch(
|
|
97
|
+
[PredictionRecord("new", 0.74)],
|
|
98
|
+
policy,
|
|
99
|
+
)
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Limitations
|
|
103
|
+
|
|
104
|
+
CalibRoute is an evaluation and routing tool, not a safety certification.
|
|
105
|
+
Thresholds should be revalidated after changes to the model, task, prompt, or
|
|
106
|
+
deployment distribution. See [design principles](docs/design.md) for details.
|
|
107
|
+
|
|
108
|
+
## Development
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
python -m unittest discover -s tests -v
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
|
|
115
|
+
Release preparation is documented in [releasing](docs/releasing.md).
|
|
116
|
+
The [Financial NER case study](examples/financial_ner/README.md) demonstrates
|
|
117
|
+
the adapter and cross-domain failure pattern on 2,098 derived prediction rows.
|
|
118
|
+
|
|
119
|
+
## License
|
|
120
|
+
|
|
121
|
+
MIT. See [LICENSE](LICENSE).
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Roadmap
|
|
2
|
+
|
|
3
|
+
## 0.1 - Core control contract
|
|
4
|
+
|
|
5
|
+
- CSV and JSONL input
|
|
6
|
+
- confidence, calibration, and risk-coverage audit
|
|
7
|
+
- validation-only threshold fitting
|
|
8
|
+
- shift-first accept/review/abstain routing
|
|
9
|
+
- deterministic tests and GitHub Actions
|
|
10
|
+
|
|
11
|
+
## 0.2 - Financial NER reference adapter
|
|
12
|
+
|
|
13
|
+
- [x] convert encoder and generative sentence outputs into the common schema
|
|
14
|
+
- [x] reproduce multi-domain confidence and risk analyses
|
|
15
|
+
- [x] add exact one-sided binomial risk bounds
|
|
16
|
+
- [x] calibrate shift thresholds by batch size and handle small batches
|
|
17
|
+
- [ ] add entity-level policies
|
|
18
|
+
- [ ] publish a complete external data acquisition manifest
|
|
19
|
+
|
|
20
|
+
## 0.3 - Correctness and reproducible evaluation
|
|
21
|
+
|
|
22
|
+
- [x] tie-aware fitting and audit operating points
|
|
23
|
+
- [x] independent frozen-policy holdout assessment
|
|
24
|
+
- [x] O(n log n) tie-aware AUROC
|
|
25
|
+
- [x] input validation, package metadata, provenance, and build checks
|
|
26
|
+
|
|
27
|
+
## Future uncertainty monitoring
|
|
28
|
+
|
|
29
|
+
- pluggable embedding and feature-shift detectors
|
|
30
|
+
- bootstrap uncertainty for audit metrics
|
|
31
|
+
- drift alerts over time windows
|
|
32
|
+
|
|
33
|
+
## 0.4 - Agent and language-system adapters
|
|
34
|
+
|
|
35
|
+
- structured-output and tool-call correctness adapters
|
|
36
|
+
- cost-sensitive escalation policies
|
|
37
|
+
- regression gates for persistent agent updates
|
|
38
|
+
- policy comparison reports
|
|
39
|
+
|
|
40
|
+
## 1.0 - Stable public API
|
|
41
|
+
|
|
42
|
+
- versioned policy and report schemas
|
|
43
|
+
- expanded documentation and examples
|
|
44
|
+
- independent reproducibility report
|
|
45
|
+
- PyPI release and archival DOI
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Security policy
|
|
2
|
+
|
|
3
|
+
CalibRoute can influence whether AI outputs are automated or reviewed. Please do
|
|
4
|
+
not disclose an exploitable routing bypass in a public issue before maintainers
|
|
5
|
+
have had a reasonable opportunity to assess it.
|
|
6
|
+
|
|
7
|
+
Until a private reporting address is published, prepare a minimal reproduction
|
|
8
|
+
without confidential data and contact a maintainer through a private channel.
|
|
9
|
+
|
|
10
|
+
CalibRoute does not execute model outputs or downstream actions. Integrators
|
|
11
|
+
should preserve that separation and apply least-privilege controls to any system
|
|
12
|
+
that consumes CalibRoute decisions.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# Design principles
|
|
2
|
+
|
|
3
|
+
## Shift first, confidence second
|
|
4
|
+
|
|
5
|
+
A model can be confidently wrong when deployment inputs differ from validation
|
|
6
|
+
data. CalibRoute therefore performs a batch-level shift check before applying an
|
|
7
|
+
instance-level confidence threshold. Severe shift results in human review rather
|
|
8
|
+
than silently trusting a threshold learned under a different regime.
|
|
9
|
+
|
|
10
|
+
## Validation-only fitting
|
|
11
|
+
|
|
12
|
+
Policies are fitted on labeled validation records. Test or production labels are
|
|
13
|
+
for evaluation, not threshold selection. The serialized policy records its
|
|
14
|
+
achieved validation risk and coverage so an operating point can be audited.
|
|
15
|
+
|
|
16
|
+
## Bounded actions
|
|
17
|
+
|
|
18
|
+
CalibRoute emits one of three explicit actions:
|
|
19
|
+
|
|
20
|
+
- `accept`: permit the configured downstream automation;
|
|
21
|
+
- `human_review`: pause automation and request qualified review;
|
|
22
|
+
- `abstain`: withhold the output when confidence is below the review band.
|
|
23
|
+
|
|
24
|
+
CalibRoute itself does not execute downstream actions. The calling application
|
|
25
|
+
decides what each action is allowed to do.
|
|
26
|
+
|
|
27
|
+
## Observable reasons
|
|
28
|
+
|
|
29
|
+
Every decision carries a stable reason code and the batch shift score. This is
|
|
30
|
+
more useful for audit than returning a binary answer without the control path.
|
|
31
|
+
|
|
32
|
+
## Current boundary
|
|
33
|
+
|
|
34
|
+
CalibRoute uses the Jensen-Shannon divergence between validation and deployment
|
|
35
|
+
confidence histograms as a transparent baseline. A confidence distribution is
|
|
36
|
+
not a complete representation of input shift. Planned adapters will add feature,
|
|
37
|
+
embedding, and task-specific shift detectors without changing the control API.
|
|
38
|
+
|
|
39
|
+
## Statistical scope
|
|
40
|
+
|
|
41
|
+
`fit_policy` evaluates only complete confidence groups: its reported accepted
|
|
42
|
+
set is exactly `confidence >= accept_threshold`. Equal scores are never split.
|
|
43
|
+
The same rule applies to audit curves. Requested coverage rounds upward to the
|
|
44
|
+
next attainable threshold; AURC is the right-step integral weighted by coverage
|
|
45
|
+
increments, not the average over unique thresholds.
|
|
46
|
+
|
|
47
|
+
The Clopper-Pearson bound is an exact one-sided binomial interval for a fixed
|
|
48
|
+
acceptance rule on independent identically distributed labeled examples.
|
|
49
|
+
Searching thresholds on those same labels invalidates a claim of nominal
|
|
50
|
+
coverage for the selected threshold. The fit bound is therefore a selection
|
|
51
|
+
diagnostic. `validate_policy` / `calibroute validate` assess a frozen threshold
|
|
52
|
+
on a separate IID holdout. The caller must enforce independence; the software
|
|
53
|
+
cannot establish it from identifiers. Repeated policy selection against the
|
|
54
|
+
holdout also invalidates the interpretation. No accepted examples means no
|
|
55
|
+
evidence and a failed validation, not zero risk.
|
|
56
|
+
|
|
57
|
+
The validation API assesses threshold acceptance before the batch shift gate.
|
|
58
|
+
It does not certify the conditional risk of the full batch-dependent pipeline,
|
|
59
|
+
future domains, correlated records, or changing models/prompts.
|
|
60
|
+
|
|
61
|
+
JSD calibration simulates batches from the empirical reference histogram. It
|
|
62
|
+
does not account for uncertainty in that reference or provide a distribution-
|
|
63
|
+
free false-alarm guarantee. The default 0.02 effect floor is a heuristic and
|
|
64
|
+
must be chosen on development data for the application. Batches smaller than
|
|
65
|
+
20 are marked `insufficient_batch` and use instance thresholds; callers may
|
|
66
|
+
instead queue them or require review. Unchanged confidence histograms can hide
|
|
67
|
+
large correctness changes. Monitoring confidence cannot replace labeled audits.
|
|
68
|
+
|
|
69
|
+
## Migrating from 0.2
|
|
70
|
+
|
|
71
|
+
Refit policies and regenerate audits. Old policies can retain thresholds selected
|
|
72
|
+
inside ties. Missing `max_js_divergence` now means adaptive monitoring, matching
|
|
73
|
+
the constructor; specify `0.10` explicitly to retain that older default.
|
|
74
|
+
Policy schema is 1.2 and audit schema is 1.1. Earlier AURC and selective-point
|
|
75
|
+
values with ties are not directly comparable to the corrected reports.
|
|
76
|
+
|
|
77
|
+
AUROC uses an O(n log n) sort and tie-aware rank aggregation. Exact binomial
|
|
78
|
+
threshold search can still be expensive for many unique scores and errors;
|
|
79
|
+
the dependency-free implementation targets modest validation sets. Use the
|
|
80
|
+
empirical mode for exploratory large audits, then validate a frozen threshold.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Release workflow
|
|
2
|
+
|
|
3
|
+
0.3.0 is the first intended PyPI release. Do not publish 0.2.0.
|
|
4
|
+
Public Git history preserves the earlier defect and its correction.
|
|
5
|
+
|
|
6
|
+
## Trusted Publishing setup
|
|
7
|
+
|
|
8
|
+
On PyPI, verify the account email and complete required two-factor setup.
|
|
9
|
+
Under account Publishing, add a pending GitHub publisher for the first release:
|
|
10
|
+
|
|
11
|
+
- PyPI project: `calibroute-ai`
|
|
12
|
+
- GitHub owner: `Garyouki`
|
|
13
|
+
- Repository: `calibroute`
|
|
14
|
+
- Workflow filename: `publish.yml`
|
|
15
|
+
- Environment: `pypi`
|
|
16
|
+
|
|
17
|
+
The workflow `.github/workflows/publish.yml` is manually dispatched from `main`.
|
|
18
|
+
It tests and builds the selected commit, then publishes the resulting artifacts
|
|
19
|
+
in a separate job with short-lived OIDC credentials. No password or API token
|
|
20
|
+
belongs in this repository. Configure environment protection on GitHub if
|
|
21
|
+
additional release approval is desired. Do not dispatch until the corresponding
|
|
22
|
+
PyPI publisher has been configured and release metadata is finalized.
|
|
23
|
+
|
|
24
|
+
## Release checks
|
|
25
|
+
|
|
26
|
+
1. Run tests, lint, the case study, and build checks below.
|
|
27
|
+
2. Check package version, `__version__`, CITATION, and changelog agree.
|
|
28
|
+
3. On actual release, set CITATION `date-released` to that release date and
|
|
29
|
+
replace the changelog's pending marker. Create an annotated tag `v0.3.0`
|
|
30
|
+
and a GitHub Release pointing at the verified commit.
|
|
31
|
+
4. Publish built artifacts using PyPI Trusted Publishing or a token configured
|
|
32
|
+
locally. Never put a token in chat, source, or a committed `.pypirc`.
|
|
33
|
+
5. Verify installation from PyPI in a fresh environment before describing the
|
|
34
|
+
package as published. Add the PyPI install command as the default README path.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
python -m pip install -e ".[dev]"
|
|
38
|
+
python -m unittest discover -s tests -v
|
|
39
|
+
ruff check src tests examples/financial_ner/run_case_study.py
|
|
40
|
+
ruff format --check src tests examples/financial_ner/run_case_study.py
|
|
41
|
+
python examples/financial_ner/run_case_study.py
|
|
42
|
+
python -m build
|
|
43
|
+
python -m twine check dist/*
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Preserve the release URL, commit, checksums, CI result, and reproducible outputs.
|
|
47
|
+
For an impact record, distinguish authored software from independent adoption:
|
|
48
|
+
record only verifiable external uses, citations, contributed issues, and feedback
|
|
49
|
+
with their dates and links. A repository or passing CI alone does not establish
|
|
50
|
+
independent use or practical impact.
|