calibroute-ai 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. calibroute_ai-0.3.0/.github/workflows/ci.yml +24 -0
  2. calibroute_ai-0.3.0/.github/workflows/publish.yml +47 -0
  3. calibroute_ai-0.3.0/.gitignore +13 -0
  4. calibroute_ai-0.3.0/CHANGELOG.md +29 -0
  5. calibroute_ai-0.3.0/CITATION.cff +11 -0
  6. calibroute_ai-0.3.0/CONTRIBUTING.md +23 -0
  7. calibroute_ai-0.3.0/LICENSE +21 -0
  8. calibroute_ai-0.3.0/PKG-INFO +168 -0
  9. calibroute_ai-0.3.0/README.md +121 -0
  10. calibroute_ai-0.3.0/ROADMAP.md +45 -0
  11. calibroute_ai-0.3.0/SECURITY.md +12 -0
  12. calibroute_ai-0.3.0/docs/design.md +80 -0
  13. calibroute_ai-0.3.0/docs/releasing.md +50 -0
  14. calibroute_ai-0.3.0/examples/financial_ner/PROVENANCE.md +32 -0
  15. calibroute_ai-0.3.0/examples/financial_ner/README.md +71 -0
  16. calibroute_ai-0.3.0/examples/financial_ner/encoder_audit.json +196 -0
  17. calibroute_ai-0.3.0/examples/financial_ner/encoder_audit.md +34 -0
  18. calibroute_ai-0.3.0/examples/financial_ner/encoder_policy.json +30 -0
  19. calibroute_ai-0.3.0/examples/financial_ner/encoder_seed42.csv +1050 -0
  20. calibroute_ai-0.3.0/examples/financial_ner/generative_audit.json +196 -0
  21. calibroute_ai-0.3.0/examples/financial_ner/generative_audit.md +34 -0
  22. calibroute_ai-0.3.0/examples/financial_ner/generative_self_consistency_seed42.csv +1050 -0
  23. calibroute_ai-0.3.0/examples/financial_ner/routing_summary.json +89 -0
  24. calibroute_ai-0.3.0/examples/financial_ner/run_case_study.py +39 -0
  25. calibroute_ai-0.3.0/examples/production_batch.csv +7 -0
  26. calibroute_ai-0.3.0/examples/validation.csv +22 -0
  27. calibroute_ai-0.3.0/pyproject.toml +52 -0
  28. calibroute_ai-0.3.0/src/calibroute/__init__.py +25 -0
  29. calibroute_ai-0.3.0/src/calibroute/adapters/__init__.py +5 -0
  30. calibroute_ai-0.3.0/src/calibroute/adapters/financial_ner.py +81 -0
  31. calibroute_ai-0.3.0/src/calibroute/bounds.py +50 -0
  32. calibroute_ai-0.3.0/src/calibroute/cli.py +135 -0
  33. calibroute_ai-0.3.0/src/calibroute/io.py +72 -0
  34. calibroute_ai-0.3.0/src/calibroute/metrics.py +145 -0
  35. calibroute_ai-0.3.0/src/calibroute/models.py +58 -0
  36. calibroute_ai-0.3.0/src/calibroute/policy.py +267 -0
  37. calibroute_ai-0.3.0/src/calibroute/report.py +63 -0
  38. calibroute_ai-0.3.0/src/calibroute/shift.py +82 -0
  39. calibroute_ai-0.3.0/src/calibroute/validation.py +46 -0
  40. calibroute_ai-0.3.0/tests/test_adapter.py +64 -0
  41. calibroute_ai-0.3.0/tests/test_bounds.py +23 -0
  42. calibroute_ai-0.3.0/tests/test_io.py +36 -0
  43. calibroute_ai-0.3.0/tests/test_metrics.py +50 -0
  44. calibroute_ai-0.3.0/tests/test_policy.py +124 -0
  45. calibroute_ai-0.3.0/tests/test_regressions.py +114 -0
  46. calibroute_ai-0.3.0/tests/test_shift.py +34 -0
@@ -0,0 +1,24 @@
1
+ name: ci
2
+
3
+ on:
4
+ push:
5
+ pull_request:
6
+
7
+ jobs:
8
+ test:
9
+ runs-on: ubuntu-latest
10
+ strategy:
11
+ matrix:
12
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
13
+ steps:
14
+ - uses: actions/checkout@v4
15
+ - uses: actions/setup-python@v5
16
+ with:
17
+ python-version: ${{ matrix.python-version }}
18
+ - run: python -m pip install -e ".[dev]"
19
+ - run: ruff check src tests examples/financial_ner/run_case_study.py
20
+ - run: ruff format --check src tests examples/financial_ner/run_case_study.py
21
+ - run: python -m unittest discover -s tests -v
22
+ - run: python examples/financial_ner/run_case_study.py
23
+ - run: python -m build
24
+ - run: python -m twine check dist/*
@@ -0,0 +1,47 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ workflow_dispatch:
5
+
6
+ permissions:
7
+ contents: read
8
+
9
+ concurrency:
10
+ group: pypi-publication
11
+ cancel-in-progress: false
12
+
13
+ jobs:
14
+ build:
15
+ if: github.ref == 'refs/heads/main'
16
+ runs-on: ubuntu-latest
17
+ steps:
18
+ - uses: actions/checkout@v4
19
+ - uses: actions/setup-python@v5
20
+ with:
21
+ python-version: "3.12"
22
+ - run: python -m pip install -e ".[dev]"
23
+ - run: python -m unittest discover -s tests -v
24
+ - run: ruff check src tests examples/financial_ner/run_case_study.py
25
+ - run: python -m build
26
+ - run: python -m twine check dist/*
27
+ - uses: actions/upload-artifact@v4
28
+ with:
29
+ name: pypi-distributions
30
+ path: dist/
31
+ if-no-files-found: error
32
+
33
+ publish:
34
+ needs: build
35
+ runs-on: ubuntu-latest
36
+ environment:
37
+ name: pypi
38
+ url: https://pypi.org/project/calibroute-ai/
39
+ permissions:
40
+ id-token: write
41
+ steps:
42
+ - uses: actions/download-artifact@v4
43
+ with:
44
+ name: pypi-distributions
45
+ path: dist/
46
+ - name: Publish verified distributions using Trusted Publishing
47
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,13 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ .pytest_cache/
5
+ .ruff_cache/
6
+ .venv/
7
+ venv/
8
+ dist/
9
+ build/
10
+ coverage.xml
11
+ .coverage
12
+ examples/output/
13
+ workcheck/
@@ -0,0 +1,29 @@
1
+ # Changelog
2
+
3
+ ## 0.3.0 - 2026-09-20
4
+
5
+ - Fix tie groups in threshold fitting, risk-coverage curves, and selective reports.
6
+ - Add independent holdout validation with CLI pass/fail exit codes.
7
+ - Clarify pointwise fit bounds versus independent fixed-policy risk assessment.
8
+ - Replace quadratic AUROC with tie-aware O(n log n) rank aggregation.
9
+ - Make missing shift threshold default to adaptive monitoring; validate inputs.
10
+ - Regenerate case-study reports and record provenance and reproducibility limits.
11
+ - Align package/citation versions; expand CI to lint, build, and check distributions.
12
+
13
+ Refit 0.2 policies before deployment. The 0.2 policy selection and audit reports
14
+ could split ties and depend on input order. No 0.2 PyPI release is intended.
15
+
16
+ ## 0.2.0 - 2026-09-19
17
+
18
+ - Add exact Clopper-Pearson upper risk bounds to policy fitting.
19
+ - Add batch-size-aware JSD calibration and explicit small-batch handling.
20
+ - Add a Financial NER sentence-output adapter and multi-domain case study.
21
+ - Expand policy schema to record declared risk and calibration settings.
22
+
23
+ ## 0.1.0 - 2026-09-19
24
+
25
+ - Initial model-agnostic confidence audit.
26
+ - Validation-grounded selective policy fitting.
27
+ - Batch confidence-shift detection.
28
+ - Auditable accept, human-review, and abstain routing.
29
+ - Dependency-free Python API and CLI.
@@ -0,0 +1,11 @@
1
+ cff-version: 1.2.0
2
+ message: "If CalibRoute supports your research, please cite the software release."
3
+ title: "CalibRoute: Uncertainty-Aware Evaluation and Decision Control for AI Systems"
4
+ type: software
5
+ version: 0.3.0
6
+ date-released: 2026-09-20
7
+ repository-code: "https://github.com/Garyouki/calibroute"
8
+ authors:
9
+ - family-names: "Zheng"
10
+ given-names: "Zihao"
11
+ license: MIT
@@ -0,0 +1,23 @@
1
+ # Contributing
2
+
3
+ Contributions are welcome, especially new metrics, shift-monitor adapters,
4
+ task-specific examples, and tests that expose unsafe assumptions.
5
+
6
+ ## Local workflow
7
+
8
+ 1. Create a focused branch.
9
+ 2. Install the package with `python -m pip install -e .`.
10
+ 3. Run `python -m unittest discover -s tests -v`.
11
+ 4. Add tests for every behavior change.
12
+ 5. Document whether a new metric is descriptive, empirically calibrated, or
13
+ accompanied by a formal guarantee.
14
+
15
+ Please do not commit private, proprietary, personally identifiable, or
16
+ license-restricted datasets. Prefer download scripts, hashes, and synthetic
17
+ fixtures for examples.
18
+
19
+ ## Pull requests
20
+
21
+ Keep pull requests narrow. Explain the uncertainty assumption, expected failure
22
+ mode, and how the change affects accept/review/abstain decisions.
23
+
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Zihao Zheng and CalibRoute contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,168 @@
1
+ Metadata-Version: 2.5
2
+ Name: calibroute-ai
3
+ Version: 0.3.0
4
+ Summary: Uncertainty-aware evaluation and decision control for AI systems.
5
+ Project-URL: Repository, https://github.com/Garyouki/calibroute
6
+ Project-URL: Issues, https://github.com/Garyouki/calibroute/issues
7
+ Project-URL: Documentation, https://github.com/Garyouki/calibroute/blob/main/README.md
8
+ Author: Zihao Zheng
9
+ License: MIT License
10
+
11
+ Copyright (c) 2026 Zihao Zheng and CalibRoute contributors
12
+
13
+ Permission is hereby granted, free of charge, to any person obtaining a copy
14
+ of this software and associated documentation files (the "Software"), to deal
15
+ in the Software without restriction, including without limitation the rights
16
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
17
+ copies of the Software, and to permit persons to whom the Software is
18
+ furnished to do so, subject to the following conditions:
19
+
20
+ The above copyright notice and this permission notice shall be included in all
21
+ copies or substantial portions of the Software.
22
+
23
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
24
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
25
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
26
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
27
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
28
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
29
+ SOFTWARE.
30
+ License-File: LICENSE
31
+ Keywords: abstention,ai-evaluation,distribution-shift,selective-prediction,trustworthy-ai,uncertainty
32
+ Classifier: Development Status :: 3 - Alpha
33
+ Classifier: License :: OSI Approved :: MIT License
34
+ Classifier: Programming Language :: Python :: 3
35
+ Classifier: Programming Language :: Python :: 3.10
36
+ Classifier: Programming Language :: Python :: 3.11
37
+ Classifier: Programming Language :: Python :: 3.12
38
+ Classifier: Programming Language :: Python :: 3.13
39
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
40
+ Requires-Python: >=3.10
41
+ Provides-Extra: dev
42
+ Requires-Dist: build>=1.2; extra == 'dev'
43
+ Requires-Dist: pytest>=8; extra == 'dev'
44
+ Requires-Dist: ruff>=0.8; extra == 'dev'
45
+ Requires-Dist: twine>=6; extra == 'dev'
46
+ Description-Content-Type: text/markdown
47
+
48
+ # CalibRoute
49
+
50
+ CalibRoute is a lightweight Python toolkit for confidence evaluation and
51
+ uncertainty-aware routing. It converts model predictions into three actions:
52
+ `accept`, `human_review`, or `abstain`.
53
+
54
+ ## Features
55
+
56
+ - Confidence calibration and error-ranking metrics
57
+ - Risk-coverage analysis
58
+ - Validation-only threshold fitting
59
+ - Tie-aware threshold fitting and independent holdout risk assessment
60
+ - Batch-size-aware confidence-shift detection
61
+ - Financial NER encoder and generative-output adapter
62
+ - CSV and JSONL support
63
+ - Dependency-free Python API and CLI
64
+
65
+ ## Install
66
+
67
+ Requires Python 3.10 or newer.
68
+
69
+ Install with `python -m pip install calibroute-ai`.
70
+
71
+ To run the checked-in examples or contribute, clone the repository:
72
+
73
+ ```bash
74
+ git clone https://github.com/Garyouki/calibroute.git
75
+ cd calibroute
76
+ python -m pip install -e .
77
+ ```
78
+
79
+ The distribution name is `calibroute-ai`; the Python import and command are
80
+ `calibroute`. No API key or model service is needed.
81
+
82
+ ## Quick start
83
+
84
+ ```bash
85
+ # Audit labeled predictions
86
+ calibroute audit \
87
+ --input examples/validation.csv \
88
+ --output examples/audit.md
89
+
90
+ # Fit an acceptance policy on validation data
91
+ calibroute fit \
92
+ --input examples/validation.csv \
93
+ --max-risk 0.20 \
94
+ --min-coverage 0.25 \
95
+ --risk-method empirical \
96
+ --output examples/policy.json
97
+
98
+ # Route a new prediction batch
99
+ calibroute route \
100
+ --input examples/production_batch.csv \
101
+ --policy examples/policy.json \
102
+ --output examples/decisions.csv \
103
+ --summary examples/routing-summary.json
104
+ ```
105
+
106
+ ## Input format
107
+
108
+ | Field | Required | Description |
109
+ |---|---|---|
110
+ | `id` | recommended | Prediction identifier |
111
+ | `confidence` | yes | Number between 0 and 1 |
112
+ | `correct` | `audit` and `fit` only | Boolean or `1`/`0` label |
113
+ | `domain` | no | Evaluation slice or deployment domain |
114
+
115
+ Additional columns are preserved as metadata.
116
+
117
+ The default fit searches thresholds using pointwise 95% Clopper-Pearson bounds.
118
+ Threshold selection on the same labels does **not** provide a 95% guarantee for
119
+ the selected policy. Freeze the policy, then assess it on a separate IID holdout:
120
+
121
+ ```bash
122
+ calibroute validate --input holdout.csv --policy examples/policy.json --output holdout-report.json
123
+ ```
124
+
125
+ Exit code 0 means the holdout upper bound meets the declared risk limit; 1 means
126
+ it does not (including zero accepted samples); 2 means invalid input. Never tune
127
+ against this holdout or reuse it to select among multiple policies. A pass does
128
+ not cover distribution shift. See [statistical scope](docs/design.md).
129
+
130
+ ## Python API
131
+
132
+ ```python
133
+ from calibroute import PredictionRecord, fit_policy, route_batch
134
+
135
+ validation = [
136
+ PredictionRecord("a", 0.98, True),
137
+ PredictionRecord("b", 0.82, True),
138
+ PredictionRecord("c", 0.55, False),
139
+ ]
140
+
141
+ policy = fit_policy(validation, max_risk=0.10, min_coverage=0.50,
142
+ risk_method="empirical") # Tiny illustrative sample only.
143
+ decisions, summary = route_batch(
144
+ [PredictionRecord("new", 0.74)],
145
+ policy,
146
+ )
147
+ ```
148
+
149
+ ## Limitations
150
+
151
+ CalibRoute is an evaluation and routing tool, not a safety certification.
152
+ Thresholds should be revalidated after changes to the model, task, prompt, or
153
+ deployment distribution. See [design principles](docs/design.md) for details.
154
+
155
+ ## Development
156
+
157
+ ```bash
158
+ python -m unittest discover -s tests -v
159
+ ```
160
+
161
+ See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
162
+ Release preparation is documented in [releasing](docs/releasing.md).
163
+ The [Financial NER case study](examples/financial_ner/README.md) demonstrates
164
+ the adapter and cross-domain failure pattern on 2,098 derived prediction rows.
165
+
166
+ ## License
167
+
168
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,121 @@
1
+ # CalibRoute
2
+
3
+ CalibRoute is a lightweight Python toolkit for confidence evaluation and
4
+ uncertainty-aware routing. It converts model predictions into three actions:
5
+ `accept`, `human_review`, or `abstain`.
6
+
7
+ ## Features
8
+
9
+ - Confidence calibration and error-ranking metrics
10
+ - Risk-coverage analysis
11
+ - Validation-only threshold fitting
12
+ - Tie-aware threshold fitting and independent holdout risk assessment
13
+ - Batch-size-aware confidence-shift detection
14
+ - Financial NER encoder and generative-output adapter
15
+ - CSV and JSONL support
16
+ - Dependency-free Python API and CLI
17
+
18
+ ## Install
19
+
20
+ Requires Python 3.10 or newer.
21
+
22
+ Install with `python -m pip install calibroute-ai`.
23
+
24
+ To run the checked-in examples or contribute, clone the repository:
25
+
26
+ ```bash
27
+ git clone https://github.com/Garyouki/calibroute.git
28
+ cd calibroute
29
+ python -m pip install -e .
30
+ ```
31
+
32
+ The distribution name is `calibroute-ai`; the Python import and command are
33
+ `calibroute`. No API key or model service is needed.
34
+
35
+ ## Quick start
36
+
37
+ ```bash
38
+ # Audit labeled predictions
39
+ calibroute audit \
40
+ --input examples/validation.csv \
41
+ --output examples/audit.md
42
+
43
+ # Fit an acceptance policy on validation data
44
+ calibroute fit \
45
+ --input examples/validation.csv \
46
+ --max-risk 0.20 \
47
+ --min-coverage 0.25 \
48
+ --risk-method empirical \
49
+ --output examples/policy.json
50
+
51
+ # Route a new prediction batch
52
+ calibroute route \
53
+ --input examples/production_batch.csv \
54
+ --policy examples/policy.json \
55
+ --output examples/decisions.csv \
56
+ --summary examples/routing-summary.json
57
+ ```
58
+
59
+ ## Input format
60
+
61
+ | Field | Required | Description |
62
+ |---|---|---|
63
+ | `id` | recommended | Prediction identifier |
64
+ | `confidence` | yes | Number between 0 and 1 |
65
+ | `correct` | `audit` and `fit` only | Boolean or `1`/`0` label |
66
+ | `domain` | no | Evaluation slice or deployment domain |
67
+
68
+ Additional columns are preserved as metadata.
69
+
70
+ The default fit searches thresholds using pointwise 95% Clopper-Pearson bounds.
71
+ Threshold selection on the same labels does **not** provide a 95% guarantee for
72
+ the selected policy. Freeze the policy, then assess it on a separate IID holdout:
73
+
74
+ ```bash
75
+ calibroute validate --input holdout.csv --policy examples/policy.json --output holdout-report.json
76
+ ```
77
+
78
+ Exit code 0 means the holdout upper bound meets the declared risk limit; 1 means
79
+ it does not (including zero accepted samples); 2 means invalid input. Never tune
80
+ against this holdout or reuse it to select among multiple policies. A pass does
81
+ not cover distribution shift. See [statistical scope](docs/design.md).
82
+
83
+ ## Python API
84
+
85
+ ```python
86
+ from calibroute import PredictionRecord, fit_policy, route_batch
87
+
88
+ validation = [
89
+ PredictionRecord("a", 0.98, True),
90
+ PredictionRecord("b", 0.82, True),
91
+ PredictionRecord("c", 0.55, False),
92
+ ]
93
+
94
+ policy = fit_policy(validation, max_risk=0.10, min_coverage=0.50,
95
+ risk_method="empirical") # Tiny illustrative sample only.
96
+ decisions, summary = route_batch(
97
+ [PredictionRecord("new", 0.74)],
98
+ policy,
99
+ )
100
+ ```
101
+
102
+ ## Limitations
103
+
104
+ CalibRoute is an evaluation and routing tool, not a safety certification.
105
+ Thresholds should be revalidated after changes to the model, task, prompt, or
106
+ deployment distribution. See [design principles](docs/design.md) for details.
107
+
108
+ ## Development
109
+
110
+ ```bash
111
+ python -m unittest discover -s tests -v
112
+ ```
113
+
114
+ See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
115
+ Release preparation is documented in [releasing](docs/releasing.md).
116
+ The [Financial NER case study](examples/financial_ner/README.md) demonstrates
117
+ the adapter and cross-domain failure pattern on 2,098 derived prediction rows.
118
+
119
+ ## License
120
+
121
+ MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,45 @@
1
+ # Roadmap
2
+
3
+ ## 0.1 - Core control contract
4
+
5
+ - CSV and JSONL input
6
+ - confidence, calibration, and risk-coverage audit
7
+ - validation-only threshold fitting
8
+ - shift-first accept/review/abstain routing
9
+ - deterministic tests and GitHub Actions
10
+
11
+ ## 0.2 - Financial NER reference adapter
12
+
13
+ - [x] convert encoder and generative sentence outputs into the common schema
14
+ - [x] reproduce multi-domain confidence and risk analyses
15
+ - [x] add exact one-sided binomial risk bounds
16
+ - [x] calibrate shift thresholds by batch size and handle small batches
17
+ - [ ] add entity-level policies
18
+ - [ ] publish a complete external data acquisition manifest
19
+
20
+ ## 0.3 - Correctness and reproducible evaluation
21
+
22
+ - [x] tie-aware fitting and audit operating points
23
+ - [x] independent frozen-policy holdout assessment
24
+ - [x] O(n log n) tie-aware AUROC
25
+ - [x] input validation, package metadata, provenance, and build checks
26
+
27
+ ## Future uncertainty monitoring
28
+
29
+ - pluggable embedding and feature-shift detectors
30
+ - bootstrap uncertainty for audit metrics
31
+ - drift alerts over time windows
32
+
33
+ ## 0.4 - Agent and language-system adapters
34
+
35
+ - structured-output and tool-call correctness adapters
36
+ - cost-sensitive escalation policies
37
+ - regression gates for persistent agent updates
38
+ - policy comparison reports
39
+
40
+ ## 1.0 - Stable public API
41
+
42
+ - versioned policy and report schemas
43
+ - expanded documentation and examples
44
+ - independent reproducibility report
45
+ - PyPI release and archival DOI
@@ -0,0 +1,12 @@
1
+ # Security policy
2
+
3
+ CalibRoute can influence whether AI outputs are automated or reviewed. Please do
4
+ not disclose an exploitable routing bypass in a public issue before maintainers
5
+ have had a reasonable opportunity to assess it.
6
+
7
+ Until a private reporting address is published, prepare a minimal reproduction
8
+ without confidential data and contact a maintainer through a private channel.
9
+
10
+ CalibRoute does not execute model outputs or downstream actions. Integrators
11
+ should preserve that separation and apply least-privilege controls to any system
12
+ that consumes CalibRoute decisions.
@@ -0,0 +1,80 @@
1
+ # Design principles
2
+
3
+ ## Shift first, confidence second
4
+
5
+ A model can be confidently wrong when deployment inputs differ from validation
6
+ data. CalibRoute therefore performs a batch-level shift check before applying an
7
+ instance-level confidence threshold. Severe shift results in human review rather
8
+ than silently trusting a threshold learned under a different regime.
9
+
10
+ ## Validation-only fitting
11
+
12
+ Policies are fitted on labeled validation records. Test or production labels are
13
+ for evaluation, not threshold selection. The serialized policy records its
14
+ achieved validation risk and coverage so an operating point can be audited.
15
+
16
+ ## Bounded actions
17
+
18
+ CalibRoute emits one of three explicit actions:
19
+
20
+ - `accept`: permit the configured downstream automation;
21
+ - `human_review`: pause automation and request qualified review;
22
+ - `abstain`: withhold the output when confidence is below the review band.
23
+
24
+ CalibRoute itself does not execute downstream actions. The calling application
25
+ decides what each action is allowed to do.
26
+
27
+ ## Observable reasons
28
+
29
+ Every decision carries a stable reason code and the batch shift score. This is
30
+ more useful for audit than returning a binary answer without the control path.
31
+
32
+ ## Current boundary
33
+
34
+ CalibRoute uses the Jensen-Shannon divergence between validation and deployment
35
+ confidence histograms as a transparent baseline. A confidence distribution is
36
+ not a complete representation of input shift. Planned adapters will add feature,
37
+ embedding, and task-specific shift detectors without changing the control API.
38
+
39
+ ## Statistical scope
40
+
41
+ `fit_policy` evaluates only complete confidence groups: its reported accepted
42
+ set is exactly `confidence >= accept_threshold`. Equal scores are never split.
43
+ The same rule applies to audit curves. Requested coverage rounds upward to the
44
+ next attainable threshold; AURC is the right-step integral weighted by coverage
45
+ increments, not the average over unique thresholds.
46
+
47
+ The Clopper-Pearson bound is an exact one-sided binomial interval for a fixed
48
+ acceptance rule on independent identically distributed labeled examples.
49
+ Searching thresholds on those same labels invalidates a claim of nominal
50
+ coverage for the selected threshold. The fit bound is therefore a selection
51
+ diagnostic. `validate_policy` / `calibroute validate` assess a frozen threshold
52
+ on a separate IID holdout. The caller must enforce independence; the software
53
+ cannot establish it from identifiers. Repeated policy selection against the
54
+ holdout also invalidates the interpretation. No accepted examples means no
55
+ evidence and a failed validation, not zero risk.
56
+
57
+ The validation API assesses threshold acceptance before the batch shift gate.
58
+ It does not certify the conditional risk of the full batch-dependent pipeline,
59
+ future domains, correlated records, or changing models/prompts.
60
+
61
+ JSD calibration simulates batches from the empirical reference histogram. It
62
+ does not account for uncertainty in that reference or provide a distribution-
63
+ free false-alarm guarantee. The default 0.02 effect floor is a heuristic and
64
+ must be chosen on development data for the application. Batches smaller than
65
+ 20 are marked `insufficient_batch` and use instance thresholds; callers may
66
+ instead queue them or require review. Unchanged confidence histograms can hide
67
+ large correctness changes. Monitoring confidence cannot replace labeled audits.
68
+
69
+ ## Migrating from 0.2
70
+
71
+ Refit policies and regenerate audits. Old policies can retain thresholds selected
72
+ inside ties. Missing `max_js_divergence` now means adaptive monitoring, matching
73
+ the constructor; specify `0.10` explicitly to retain that older default.
74
+ Policy schema is 1.2 and audit schema is 1.1. Earlier AURC and selective-point
75
+ values with ties are not directly comparable to the corrected reports.
76
+
77
+ AUROC uses an O(n log n) sort and tie-aware rank aggregation. Exact binomial
78
+ threshold search can still be expensive for many unique scores and errors;
79
+ the dependency-free implementation targets modest validation sets. Use the
80
+ empirical mode for exploratory large audits, then validate a frozen threshold.
@@ -0,0 +1,50 @@
1
+ # Release workflow
2
+
3
+ 0.3.0 is the first intended PyPI release. Do not publish 0.2.0.
4
+ Public Git history preserves the earlier defect and its correction.
5
+
6
+ ## Trusted Publishing setup
7
+
8
+ On PyPI, verify the account email and complete required two-factor setup.
9
+ Under account Publishing, add a pending GitHub publisher for the first release:
10
+
11
+ - PyPI project: `calibroute-ai`
12
+ - GitHub owner: `Garyouki`
13
+ - Repository: `calibroute`
14
+ - Workflow filename: `publish.yml`
15
+ - Environment: `pypi`
16
+
17
+ The workflow `.github/workflows/publish.yml` is manually dispatched from `main`.
18
+ It tests and builds the selected commit, then publishes the resulting artifacts
19
+ in a separate job with short-lived OIDC credentials. No password or API token
20
+ belongs in this repository. Configure environment protection on GitHub if
21
+ additional release approval is desired. Do not dispatch until the corresponding
22
+ PyPI publisher has been configured and release metadata is finalized.
23
+
24
+ ## Release checks
25
+
26
+ 1. Run tests, lint, the case study, and build checks below.
27
+ 2. Check package version, `__version__`, CITATION, and changelog agree.
28
+ 3. On actual release, set CITATION `date-released` to that release date and
29
+ replace the changelog's pending marker. Create an annotated tag `v0.3.0`
30
+ and a GitHub Release pointing at the verified commit.
31
+ 4. Publish built artifacts using PyPI Trusted Publishing or a token configured
32
+ locally. Never put a token in chat, source, or a committed `.pypirc`.
33
+ 5. Verify installation from PyPI in a fresh environment before describing the
34
+ package as published. Add the PyPI install command as the default README path.
35
+
36
+ ```bash
37
+ python -m pip install -e ".[dev]"
38
+ python -m unittest discover -s tests -v
39
+ ruff check src tests examples/financial_ner/run_case_study.py
40
+ ruff format --check src tests examples/financial_ner/run_case_study.py
41
+ python examples/financial_ner/run_case_study.py
42
+ python -m build
43
+ python -m twine check dist/*
44
+ ```
45
+
46
+ Preserve the release URL, commit, checksums, CI result, and reproducible outputs.
47
+ For an impact record, distinguish authored software from independent adoption:
48
+ record only verifiable external uses, citations, contributed issues, and feedback
49
+ with their dates and links. A repository or passing CI alone does not establish
50
+ independent use or practical impact.