cci-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cci_sdk-0.1.0.dist-info/METADATA +430 -0
- cci_sdk-0.1.0.dist-info/RECORD +92 -0
- cci_sdk-0.1.0.dist-info/WHEEL +4 -0
- cci_sdk-0.1.0.dist-info/entry_points.txt +2 -0
- cci_sdk-0.1.0.dist-info/licenses/LICENSE +202 -0
- cli_sdk/__init__.py +127 -0
- cli_sdk/answers/__init__.py +29 -0
- cli_sdk/answers/base.py +28 -0
- cli_sdk/answers/belief_answer.py +45 -0
- cli_sdk/answers/claim_answer.py +53 -0
- cli_sdk/answers/gate_answer.py +33 -0
- cli_sdk/answers/guarantee.py +163 -0
- cli_sdk/answers/interval_answer.py +34 -0
- cli_sdk/answers/judge_answer.py +30 -0
- cli_sdk/answers/response.py +130 -0
- cli_sdk/answers/route_answer.py +29 -0
- cli_sdk/answers/set_answer.py +45 -0
- cli_sdk/backends/__init__.py +37 -0
- cli_sdk/backends/anthropic.py +22 -0
- cli_sdk/backends/azure_openai.py +22 -0
- cli_sdk/backends/base.py +192 -0
- cli_sdk/backends/bedrock.py +20 -0
- cli_sdk/backends/custom.py +5 -0
- cli_sdk/backends/gemini.py +18 -0
- cli_sdk/backends/openai.py +22 -0
- cli_sdk/backends/openrouter.py +23 -0
- cli_sdk/backends/sglang.py +29 -0
- cli_sdk/backends/vllm.py +31 -0
- cli_sdk/calibration/__init__.py +28 -0
- cli_sdk/calibration/audit.py +94 -0
- cli_sdk/calibration/examples.py +55 -0
- cli_sdk/calibration/label_efficient.py +78 -0
- cli_sdk/calibration/profile.py +236 -0
- cli_sdk/cli_tool.py +151 -0
- cli_sdk/client/__init__.py +7 -0
- cli_sdk/client/async_client.py +113 -0
- cli_sdk/client/base.py +207 -0
- cli_sdk/client/retries.py +72 -0
- cli_sdk/client/sync_client.py +140 -0
- cli_sdk/constants.py +31 -0
- cli_sdk/evidence/__init__.py +45 -0
- cli_sdk/evidence/_prompts.py +165 -0
- cli_sdk/evidence/anthropic.py +140 -0
- cli_sdk/evidence/base.py +121 -0
- cli_sdk/evidence/langchain.py +114 -0
- cli_sdk/evidence/mock.py +133 -0
- cli_sdk/evidence/openai_compatible.py +278 -0
- cli_sdk/exceptions.py +60 -0
- cli_sdk/integrations/__init__.py +33 -0
- cli_sdk/integrations/_guard.py +356 -0
- cli_sdk/integrations/agent_framework.py +140 -0
- cli_sdk/integrations/google_adk.py +148 -0
- cli_sdk/integrations/langchain.py +251 -0
- cli_sdk/integrations/langgraph.py +216 -0
- cli_sdk/local/__init__.py +14 -0
- cli_sdk/local/client.py +532 -0
- cli_sdk/local/engine.py +645 -0
- cli_sdk/local/scoring.py +170 -0
- cli_sdk/local/store.py +151 -0
- cli_sdk/monitoring/__init__.py +6 -0
- cli_sdk/monitoring/alerts.py +47 -0
- cli_sdk/monitoring/monitor.py +180 -0
- cli_sdk/py.typed +0 -0
- cli_sdk/queries/__init__.py +15 -0
- cli_sdk/queries/base.py +59 -0
- cli_sdk/queries/belief.py +25 -0
- cli_sdk/queries/claim.py +27 -0
- cli_sdk/queries/gate.py +38 -0
- cli_sdk/queries/interval.py +33 -0
- cli_sdk/queries/judge.py +27 -0
- cli_sdk/queries/route.py +35 -0
- cli_sdk/queries/set.py +35 -0
- cli_sdk/stats/__init__.py +25 -0
- cli_sdk/stats/conformal/__init__.py +28 -0
- cli_sdk/stats/conformal/_quantile.py +144 -0
- cli_sdk/stats/conformal/aps.py +98 -0
- cli_sdk/stats/conformal/cqr.py +42 -0
- cli_sdk/stats/conformal/crc.py +71 -0
- cli_sdk/stats/conformal/lac.py +37 -0
- cli_sdk/stats/conformal/ltt.py +77 -0
- cli_sdk/stats/conformal/mondrian.py +72 -0
- cli_sdk/stats/conformal/raps.py +85 -0
- cli_sdk/stats/conformal/rcps.py +103 -0
- cli_sdk/stats/evalues/__init__.py +20 -0
- cli_sdk/stats/evalues/anytime.py +124 -0
- cli_sdk/stats/evalues/ebh.py +53 -0
- cli_sdk/stats/evalues/eprocess.py +55 -0
- cli_sdk/stats/evalues/ppi.py +91 -0
- cli_sdk/stats/venn_abers/__init__.py +11 -0
- cli_sdk/stats/venn_abers/_isotonic.py +39 -0
- cli_sdk/stats/venn_abers/cvap.py +62 -0
- cli_sdk/stats/venn_abers/ivap.py +104 -0
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: cci-sdk
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python SDK and standalone statistics engine for Conformal Logit Inference (CLI) — conformal prediction, Venn-Abers calibration, and e-values for LLM decisions.
|
|
5
|
+
Project-URL: Homepage, https://cci.gitdate.ink
|
|
6
|
+
Project-URL: Documentation, https://cci-docs.gitdate.ink
|
|
7
|
+
Project-URL: Repository, https://github.com/rahvis/cci-sdk
|
|
8
|
+
Author-email: CCI <rahul.vishwakarma@workonward.org>
|
|
9
|
+
License: Apache-2.0
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: calibration,conformal-prediction,e-values,llm,uncertainty-quantification,venn-abers
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.9
|
|
18
|
+
Requires-Dist: httpx>=0.27
|
|
19
|
+
Requires-Dist: numpy>=1.24
|
|
20
|
+
Provides-Extra: agent-framework
|
|
21
|
+
Requires-Dist: agent-framework-core>=1.19; extra == 'agent-framework'
|
|
22
|
+
Provides-Extra: all
|
|
23
|
+
Requires-Dist: agent-framework-core>=1.19; extra == 'all'
|
|
24
|
+
Requires-Dist: anthropic>=1.0; extra == 'all'
|
|
25
|
+
Requires-Dist: google-adk>=2.0; extra == 'all'
|
|
26
|
+
Requires-Dist: langchain-core>=1.0; extra == 'all'
|
|
27
|
+
Requires-Dist: langchain>=1.0; extra == 'all'
|
|
28
|
+
Requires-Dist: langgraph>=1.0; extra == 'all'
|
|
29
|
+
Requires-Dist: openai>=1.40; extra == 'all'
|
|
30
|
+
Provides-Extra: anthropic
|
|
31
|
+
Requires-Dist: anthropic>=1.0; extra == 'anthropic'
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
36
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
37
|
+
Provides-Extra: google-adk
|
|
38
|
+
Requires-Dist: google-adk>=2.0; extra == 'google-adk'
|
|
39
|
+
Provides-Extra: langchain
|
|
40
|
+
Requires-Dist: langchain-core>=1.0; extra == 'langchain'
|
|
41
|
+
Requires-Dist: langchain>=1.0; extra == 'langchain'
|
|
42
|
+
Requires-Dist: langgraph>=1.0; extra == 'langchain'
|
|
43
|
+
Provides-Extra: langgraph
|
|
44
|
+
Requires-Dist: langchain-core>=1.0; extra == 'langgraph'
|
|
45
|
+
Requires-Dist: langgraph>=1.0; extra == 'langgraph'
|
|
46
|
+
Provides-Extra: openai
|
|
47
|
+
Requires-Dist: openai>=1.40; extra == 'openai'
|
|
48
|
+
Description-Content-Type: text/markdown
|
|
49
|
+
|
|
50
|
+
# cci-sdk — Python SDK for Conformal Logit Inference
|
|
51
|
+
|
|
52
|
+
Conformal Logit Inference (CLI) turns the raw output of any LLM into
|
|
53
|
+
finite-sample statistical guarantees. This package contains:
|
|
54
|
+
|
|
55
|
+
- **`cli_sdk`** — sync and async clients for the hosted API, the seven
|
|
56
|
+
guarantee-bearing query primitives, typed answers, calibration profiles,
|
|
57
|
+
and drift monitors.
|
|
58
|
+
- **`cli_sdk.stats`** — the standalone statistics engine (conformal
|
|
59
|
+
prediction, Venn-Abers calibration, e-values). It needs only NumPy and
|
|
60
|
+
makes no network calls, so it runs inside air-gapped environments on
|
|
61
|
+
precomputed score arrays.
|
|
62
|
+
|
|
63
|
+
- **`cli_sdk.local`** and **`cli_sdk.evidence`** — local mode: calibrate and
|
|
64
|
+
evaluate the same queries in-process on your own model (OpenAI, Azure
|
|
65
|
+
OpenAI, Anthropic, Gemini, or open weights such as Gemma on vLLM or
|
|
66
|
+
SGLang), with calibration profiles stored as reviewable JSON files.
|
|
67
|
+
- **`cli_sdk.integrations`** — calibrated guardrails for agent tool calls in
|
|
68
|
+
LangChain, LangGraph, Google ADK and Microsoft Agent Framework: allow,
|
|
69
|
+
escalate to a human, or block, each with a stated guarantee.
|
|
70
|
+
|
|
71
|
+
Every guarantee is marginal or group-conditional over data exchangeable with
|
|
72
|
+
a stated calibration set. None is a promise about one decision in isolation.
|
|
73
|
+
|
|
74
|
+
## Install
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
pip install cci-sdk # or: uv add cli-sdk
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Optional extras for model providers and agent frameworks (Python 3.10+ for
|
|
81
|
+
the frameworks):
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
pip install "cci-sdk[openai]" # OpenAI, Azure OpenAI, vLLM, SGLang evidence backends
|
|
85
|
+
pip install "cci-sdk[anthropic]" # Claude evidence backend
|
|
86
|
+
pip install "cci-sdk[langchain,openai]" # or [langgraph], [google-adk], [agent-framework], [all]
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
For development in this repository:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
cd sdk/python
|
|
93
|
+
pip install -e ".[dev]"
|
|
94
|
+
pytest
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Authenticate
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
export CLI_API_KEY=sk_live_...
|
|
101
|
+
# optional, for a self-hosted deployment:
|
|
102
|
+
export CLI_BASE_URL=https://cli.internal.example.com/v1
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
A key is not required when `CLI_BASE_URL` points at `localhost`.
|
|
106
|
+
|
|
107
|
+
## Accessing the API
|
|
108
|
+
|
|
109
|
+
### Over HTTP
|
|
110
|
+
|
|
111
|
+
Every call is `POST https://cci.gitdate.ink/api/v1/evaluate` with a bearer token:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
curl https://cci.gitdate.ink/api/v1/evaluate \
|
|
115
|
+
-H "Authorization: Bearer $CLI_API_KEY" \
|
|
116
|
+
-H "Content-Type: application/json" \
|
|
117
|
+
-d '{
|
|
118
|
+
"context": {"ticket": "My payouts have been failing for 3 days."},
|
|
119
|
+
"backend": {"provider": "openai", "model": "gpt-4.1-2025-04-14"},
|
|
120
|
+
"queries": {
|
|
121
|
+
"department": {
|
|
122
|
+
"type": "set",
|
|
123
|
+
"instructions": "Which team should handle this ticket?",
|
|
124
|
+
"options": {"billing": null, "technical": null, "sales": null},
|
|
125
|
+
"calibration_profile": "support-routing-v3",
|
|
126
|
+
"alpha": 0.10
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}'
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
| Endpoint | Purpose |
|
|
133
|
+
|---|---|
|
|
134
|
+
| `POST /v1/evaluate` | Evaluate a context against one or more queries |
|
|
135
|
+
| `POST /v1/calibration-profiles` | Create a calibration profile |
|
|
136
|
+
| `GET /v1/calibration-profiles/{name}` | Size, coverage interval, audit history |
|
|
137
|
+
| `POST /v1/calibration-profiles/{name}/examples` | Add labelled examples |
|
|
138
|
+
| `POST /v1/calibration-profiles/{name}/label-with-judge` | Judge-labelled pool plus a human sample, combined validly |
|
|
139
|
+
| `POST /v1/calibration-profiles/{name}/audit` | Audit against fresh labelled examples |
|
|
140
|
+
| `POST /v1/calibration-profiles/{name}/monitors` | Create an anytime-valid drift monitor |
|
|
141
|
+
| `GET /v1/calibration-profiles/{name}/monitors/{id}/alerts` | Poll monitor alerts |
|
|
142
|
+
| `GET /v1/backends` | Configured backends and their detected access level |
|
|
143
|
+
|
|
144
|
+
### With the SDK
|
|
145
|
+
|
|
146
|
+
```python
|
|
147
|
+
from cli_sdk import CLIClient, Set, Gate
|
|
148
|
+
|
|
149
|
+
with CLIClient() as client:
|
|
150
|
+
result = client.evaluate(
|
|
151
|
+
context={"ticket": "My payouts have been failing for 3 days."},
|
|
152
|
+
backend={"provider": "openai", "model": "gpt-4.1-2025-04-14"},
|
|
153
|
+
queries={
|
|
154
|
+
"department": Set(
|
|
155
|
+
instructions="Which team should handle this ticket?",
|
|
156
|
+
options={"billing": None, "technical": None, "sales": None},
|
|
157
|
+
calibration_profile="support-routing-v3",
|
|
158
|
+
alpha=0.10,
|
|
159
|
+
),
|
|
160
|
+
"route": Gate(
|
|
161
|
+
instructions="Auto-route this ticket without human review?",
|
|
162
|
+
calibration_profile="support-routing-v3",
|
|
163
|
+
guarantee="fdr",
|
|
164
|
+
target=0.05,
|
|
165
|
+
),
|
|
166
|
+
},
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
department = result.answers["department"]
|
|
170
|
+
print(department.set, department.guarantee.describe())
|
|
171
|
+
|
|
172
|
+
if result.answers["route"].approved:
|
|
173
|
+
assign_queue(department.top)
|
|
174
|
+
else:
|
|
175
|
+
send_to_human_triage(department.set)
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
The async client has the same interface, with `await` on each call (`monitors.poll` returns a list rather than a generator):
|
|
179
|
+
|
|
180
|
+
```python
|
|
181
|
+
from cli_sdk import AsyncCLIClient
|
|
182
|
+
|
|
183
|
+
async with AsyncCLIClient() as client:
|
|
184
|
+
result = await client.evaluate(context=..., backend=..., queries=...)
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
## The seven primitives
|
|
188
|
+
|
|
189
|
+
| Primitive | Returns | Guarantee |
|
|
190
|
+
|---|---|---|
|
|
191
|
+
| `Belief` | Venn-Abers interval `[p0, p1]` for one statement | calibrated probability bracket |
|
|
192
|
+
| `Set` | a set of options | coverage >= 1 - alpha |
|
|
193
|
+
| `Interval` | a conformalized ordinal/continuous interval | coverage >= 1 - alpha |
|
|
194
|
+
| `Gate` | `auto_approve` / `escalate` / `abstain` | risk <= alpha, risk with probability 1 - delta, or batch FDR <= q |
|
|
195
|
+
| `Claim` | long-form output filtered to supported claims | P(all retained claims true) >= 1 - alpha |
|
|
196
|
+
| `Judge` | a verdict with an escalation cascade | human agreement >= 1 - alpha on accepted verdicts |
|
|
197
|
+
| `Route` | which backend in a cascade served the request | cost or accuracy bound |
|
|
198
|
+
|
|
199
|
+
Every answer carries `answer.guarantee`, a `Guarantee` with `type`,
|
|
200
|
+
`method`, `alpha`/`delta`/`target`, `calibration_profile`, `calibration_n`,
|
|
201
|
+
`coverage_ci`, and `last_audited`. `guarantee.is_heuristic` is `True` when an
|
|
202
|
+
answer carries no formal guarantee, for example because its profile is still
|
|
203
|
+
below the minimum size. Pass `strict_guarantees=True` to the client to raise
|
|
204
|
+
`InsufficientCalibrationError` instead of receiving heuristic answers.
|
|
205
|
+
|
|
206
|
+
## Typed responses
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from cli_sdk import EvaluateResponse, SetAnswer
|
|
210
|
+
|
|
211
|
+
class RoutingResponse(EvaluateResponse):
|
|
212
|
+
department: SetAnswer
|
|
213
|
+
|
|
214
|
+
result = client.evaluate(..., response_model=RoutingResponse)
|
|
215
|
+
result.department # a SetAnswer; a missing or mistyped answer raises at parse time
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
## Calibration profiles
|
|
219
|
+
|
|
220
|
+
```python
|
|
221
|
+
client.calibration_profiles.create(
|
|
222
|
+
name="support-routing-v3",
|
|
223
|
+
backend={"provider": "openai", "model": "gpt-4.1-2025-04-14"},
|
|
224
|
+
method="APS",
|
|
225
|
+
alpha=0.10,
|
|
226
|
+
group_by="account_tier", # optional Mondrian (per-group) calibration
|
|
227
|
+
)
|
|
228
|
+
client.calibration_profiles.add_examples(
|
|
229
|
+
"support-routing-v3",
|
|
230
|
+
examples=[{"context": {"ticket": "I was charged twice."}, "label": "billing"}],
|
|
231
|
+
)
|
|
232
|
+
profile = client.calibration_profiles.get("support-routing-v3")
|
|
233
|
+
profile.n, profile.minimum_n, profile.recommended_n, profile.realized_coverage_ci
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
| alpha | hard minimum n | recommended n |
|
|
237
|
+
|---|---|---|
|
|
238
|
+
| 0.20 | 4 | ~300 |
|
|
239
|
+
| 0.10 | 9 | ~1,000 |
|
|
240
|
+
| 0.05 | 19 | ~1,500 |
|
|
241
|
+
| 0.01 | 99 | ~2,500 |
|
|
242
|
+
|
|
243
|
+
Label-efficient calibration labels a pool with a judge and a random human
|
|
244
|
+
subset, combined so the result is valid regardless of judge quality:
|
|
245
|
+
|
|
246
|
+
```python
|
|
247
|
+
client.calibration_profiles.label_with_judge(
|
|
248
|
+
"support-routing-v3",
|
|
249
|
+
judge={"provider": "openai", "model": "gpt-4.1-2025-04-14"},
|
|
250
|
+
unlabelled_examples=pool,
|
|
251
|
+
human_labelled_sample_size=300,
|
|
252
|
+
)
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
## Drift monitoring
|
|
256
|
+
|
|
257
|
+
```python
|
|
258
|
+
client.calibration_profiles.monitors.create(
|
|
259
|
+
"support-routing-v3", type="coverage", target=0.90,
|
|
260
|
+
false_alarm_rate=0.05, labelled_sample_rate=0.02,
|
|
261
|
+
)
|
|
262
|
+
for alert in client.calibration_profiles.monitors.poll("support-routing-v3"):
|
|
263
|
+
pause_auto_actioning()
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
Monitors are e-processes: the false-alarm rate holds no matter how often or
|
|
267
|
+
for how long you check them. `LocalMonitor` runs the same test in-process:
|
|
268
|
+
|
|
269
|
+
```python
|
|
270
|
+
from cli_sdk import LocalMonitor
|
|
271
|
+
|
|
272
|
+
monitor = LocalMonitor(type="coverage", target=0.90, false_alarm_rate=0.05)
|
|
273
|
+
for label, prediction_set in stream:
|
|
274
|
+
if alert := monitor.update(label in prediction_set):
|
|
275
|
+
page_oncall(alert)
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## Backends
|
|
279
|
+
|
|
280
|
+
```python
|
|
281
|
+
from cli_sdk import OpenAIBackend, AnthropicBackend, VLLMBackend
|
|
282
|
+
|
|
283
|
+
OpenAIBackend(model="gpt-4.1-2025-04-14") # L1 on non-reasoning configs
|
|
284
|
+
AnthropicBackend(model="claude-sonnet-5", sample_count=20) # L0: sampling only
|
|
285
|
+
VLLMBackend(model="Qwen/Qwen3-8B", base_url="http://vllm:8000") # up to L4
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
Plain dicts (`{"provider": "openai", "model": ...}`) work anywhere a backend
|
|
289
|
+
is accepted.
|
|
290
|
+
|
|
291
|
+
### Bring your own model
|
|
292
|
+
|
|
293
|
+
A `CustomBackend` runs your model locally and sends only the evidence
|
|
294
|
+
(probabilities, samples, scores) to CLI. Implement the methods your access
|
|
295
|
+
level supports:
|
|
296
|
+
|
|
297
|
+
```python
|
|
298
|
+
from cli_sdk import CLIClient, CustomBackend
|
|
299
|
+
|
|
300
|
+
class MyEngine(CustomBackend):
|
|
301
|
+
access_level = "L1"
|
|
302
|
+
|
|
303
|
+
def score_options(self, context, instructions, options):
|
|
304
|
+
return my_model.option_probabilities(context, instructions, list(options))
|
|
305
|
+
|
|
306
|
+
def sample(self, context, instructions, n):
|
|
307
|
+
return [my_model.generate(context, instructions) for _ in range(n)]
|
|
308
|
+
|
|
309
|
+
client = CLIClient(backend=MyEngine())
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
## Local mode
|
|
313
|
+
|
|
314
|
+
`LocalCLIClient` runs the same queries fully in-process: it scores labelled
|
|
315
|
+
examples with a model you bring, stores the calibration profile as a JSON
|
|
316
|
+
file, and answers with the same typed answers and guarantee cards as the
|
|
317
|
+
hosted client. Nothing leaves your process except calls to your own model
|
|
318
|
+
provider.
|
|
319
|
+
|
|
320
|
+
```python
|
|
321
|
+
from cli_sdk import Gate, LocalCLIClient
|
|
322
|
+
from cli_sdk.evidence import OpenAIEvidenceBackend, vllm_backend
|
|
323
|
+
|
|
324
|
+
evidence = OpenAIEvidenceBackend("gpt-4.1-mini", api_key="YOUR_OPENAI_API_KEY")
|
|
325
|
+
# or open weights: evidence = vllm_backend("google/gemma-4-12B-it", base_url="http://localhost:8000/v1")
|
|
326
|
+
|
|
327
|
+
client = LocalCLIClient(evidence, store=".cli_profiles")
|
|
328
|
+
refund = Gate(instructions="Is issuing this refund correct under the policy?",
|
|
329
|
+
calibration_profile="refund-approvals-v1", guarantee="risk", target=0.05)
|
|
330
|
+
client.calibrate(refund, labelled_examples) # [{"context": {...}, "label": True}, ...]
|
|
331
|
+
answer = client.evaluate(context=case, queries={"refund": refund}).answers["refund"]
|
|
332
|
+
print(answer.decision, answer.guarantee.describe())
|
|
333
|
+
```
|
|
334
|
+
|
|
335
|
+
Profiles are fingerprinted with the backend configuration and the query's
|
|
336
|
+
prompt: change either and answers fall back to their safe heuristic form
|
|
337
|
+
(escalate, every option, `[0, 1]`) until you recalibrate. See the
|
|
338
|
+
[local mode guide](https://cci-docs.gitdate.ink/agents/local-mode).
|
|
339
|
+
|
|
340
|
+
## Agent frameworks
|
|
341
|
+
|
|
342
|
+
`ToolGuard` decides each proposed tool call from a calibrated answer and
|
|
343
|
+
maps it onto the framework's native human-in-the-loop mechanism:
|
|
344
|
+
|
|
345
|
+
```python
|
|
346
|
+
from cli_sdk.integrations import GuardRule, ToolGuard
|
|
347
|
+
from cli_sdk.integrations.langchain import cli_middleware
|
|
348
|
+
|
|
349
|
+
guard = ToolGuard(client, [GuardRule(tool="issue_refund", query=refund, context=refund_context)])
|
|
350
|
+
agent = create_agent(model, tools=[lookup_order, issue_refund],
|
|
351
|
+
middleware=cli_middleware(guard), checkpointer=InMemorySaver())
|
|
352
|
+
```
|
|
353
|
+
|
|
354
|
+
| Framework | Module | Hook | Human review |
|
|
355
|
+
|---|---|---|---|
|
|
356
|
+
| LangChain 1.x | `cli_sdk.integrations.langchain` | `wrap_tool_call` middleware | `HumanInTheLoopMiddleware` |
|
|
357
|
+
| LangGraph | `cli_sdk.integrations.langgraph` | guard node before `ToolNode` | `interrupt()` / `Command(resume=...)` |
|
|
358
|
+
| Google ADK 2.x | `cli_sdk.integrations.google_adk` | `before_tool_callback` or plugin | tool confirmation |
|
|
359
|
+
| Microsoft Agent Framework | `cli_sdk.integrations.agent_framework` | `FunctionMiddleware` | function approval requests |
|
|
360
|
+
|
|
361
|
+
Eight runnable examples in risk-sensitive domains (refunds, patient triage,
|
|
362
|
+
insurance claims, discharge summaries, AML holds, trial screening, credit
|
|
363
|
+
tiers, drug safety) are in `examples/agents/`. Each runs offline with
|
|
364
|
+
`--provider mock` and with your own keys for OpenAI, Azure, Anthropic,
|
|
365
|
+
Gemini, or Gemma on vLLM or SGLang. See the
|
|
366
|
+
[agent frameworks guide](https://cci-docs.gitdate.ink/agents).
|
|
367
|
+
|
|
368
|
+
## Offline statistics engine
|
|
369
|
+
|
|
370
|
+
```python
|
|
371
|
+
from cli_sdk.stats.conformal import aps, crc
|
|
372
|
+
from cli_sdk.stats.venn_abers import ivap
|
|
373
|
+
from cli_sdk.stats.evalues import ebh, ppi, CoverageMonitor
|
|
374
|
+
|
|
375
|
+
q_hat = aps.calibrate(cal_probs, cal_labels, alpha=0.10)
|
|
376
|
+
sets = aps.predict(test_probs, q_hat)
|
|
377
|
+
|
|
378
|
+
p0, p1 = ivap.calibrate_and_predict(cal_scores, cal_labels, test_scores)
|
|
379
|
+
|
|
380
|
+
selected = ebh.select(e_values, q=0.10)
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
| Module | Methods |
|
|
384
|
+
|---|---|
|
|
385
|
+
| `stats.conformal` | LAC, APS, RAPS, CQR, CRC, RCPS, Learn-then-Test, Mondrian |
|
|
386
|
+
| `stats.venn_abers` | IVAP, CVAP, interval merging and width |
|
|
387
|
+
| `stats.evalues` | anytime-valid risk and coverage monitors, e-BH, prediction-powered inference |
|
|
388
|
+
|
|
389
|
+
## Command-line tool
|
|
390
|
+
|
|
391
|
+
```bash
|
|
392
|
+
cli calibration show support-routing-v3
|
|
393
|
+
cli calibration audit support-routing-v3 --examples fresh.jsonl --fail-below 0.88
|
|
394
|
+
cli calibration audit-local --covered outcomes.txt --target 0.90
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
`audit` exits with status 1 on failure, so it can block a deploy in CI.
|
|
398
|
+
|
|
399
|
+
## Errors
|
|
400
|
+
|
|
401
|
+
| Exception | When |
|
|
402
|
+
|---|---|
|
|
403
|
+
| `AuthenticationError` | 401: missing or invalid API key |
|
|
404
|
+
| `ValidationError` | 422: the request failed validation |
|
|
405
|
+
| `InsufficientCalibrationError` | a profile is too small for the guarantee (raised only with `strict_guarantees=True`) |
|
|
406
|
+
| `RateLimitError` | 429 after retries are exhausted |
|
|
407
|
+
| `BackendError` | 502 / 529: the model backend errored or reported a lower access level than required |
|
|
408
|
+
| `ConfigurationError` | client-side misconfiguration, raised before any request |
|
|
409
|
+
|
|
410
|
+
`429`, `502` and `529` are retried with full-jitter exponential backoff,
|
|
411
|
+
honoring `retry-after`. Configure with `CLIClient(retry=RetryConfig(max_attempts=5, base_delay=0.5))`.
|
|
412
|
+
|
|
413
|
+
## Package layout
|
|
414
|
+
|
|
415
|
+
```
|
|
416
|
+
src/cli_sdk/
|
|
417
|
+
client/ sync + async clients, retry policy
|
|
418
|
+
queries/ Belief, Set, Interval, Gate, Claim, Judge, Route
|
|
419
|
+
answers/ typed answers, the Guarantee card, EvaluateResponse
|
|
420
|
+
calibration/ profiles, examples, audits, label-efficient calibration
|
|
421
|
+
monitoring/ hosted monitors, alerts, LocalMonitor
|
|
422
|
+
backends/ one adapter per provider, plus CustomBackend
|
|
423
|
+
stats/ offline engine: conformal/, venn_abers/, evalues/
|
|
424
|
+
local/ LocalCLIClient: in-process calibration and evaluation
|
|
425
|
+
evidence/ evidence backends: OpenAI, Azure, Anthropic, Gemini, vLLM, SGLang, mock
|
|
426
|
+
integrations/ ToolGuard and adapters for LangChain, LangGraph, ADK, Agent Framework
|
|
427
|
+
cli_tool.py the `cli` command
|
|
428
|
+
examples/ runnable end-to-end examples (agents/: agent-framework examples)
|
|
429
|
+
tests/ engine and client tests
|
|
430
|
+
```
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
cli_sdk/__init__.py,sha256=NKRmlmMkj-qMqknHeJFnc9t8EdiDvn6zoKY6MdNlS4k,2988
|
|
2
|
+
cli_sdk/cli_tool.py,sha256=uE36rN-Y4wHu8rffXlNvX2MGav6I6fgPuEIX9ROUM6M,5849
|
|
3
|
+
cli_sdk/constants.py,sha256=86Of269FcxCAddaeotp-oBJXZ3bg6hnQpsXvYYFY1Ok,1048
|
|
4
|
+
cli_sdk/exceptions.py,sha256=kwGHhJ3sbt3E2_tVBC7DTLRP5i_x1p3nLXGXsJOqD3w,2002
|
|
5
|
+
cli_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
cli_sdk/answers/__init__.py,sha256=ZJ637QK1TEClnoSrcJgUOiMyLGImxvOlSvkDLgZrxO0,910
|
|
7
|
+
cli_sdk/answers/base.py,sha256=QtQ4SRehQY596dSn6SVWBCS4OmjA48jwQxbemZnACKM,634
|
|
8
|
+
cli_sdk/answers/belief_answer.py,sha256=M0RcRB_ZMDwHR9-B_phzcJ4NP10ZNDngR-bOuS1PVgc,1603
|
|
9
|
+
cli_sdk/answers/claim_answer.py,sha256=DKR4GY6dFxFBRCpRUgyyECxzDfQFQYg9rye4ySvosTc,1727
|
|
10
|
+
cli_sdk/answers/gate_answer.py,sha256=YduF85j3jPyfTn3Zx7SjrsNslG9euf5WSbRMV83j4wc,963
|
|
11
|
+
cli_sdk/answers/guarantee.py,sha256=AM5l1Y0IaI1rgZ8KRrIAbsO2wGDHWhqGSlo3F30qtC0,6648
|
|
12
|
+
cli_sdk/answers/interval_answer.py,sha256=8sGmnLtj2W0hm4hM7wbavZQUv9wi87F7RXOn-TlL7pA,1088
|
|
13
|
+
cli_sdk/answers/judge_answer.py,sha256=3muSNv4aQnhJmROkeapR_qKlCSBzWPNykEI9UJiXgS8,894
|
|
14
|
+
cli_sdk/answers/response.py,sha256=h4P9vzrEI2DL5rLQ67LkQKosfvDQGI6cH1L7cOie_vo,4821
|
|
15
|
+
cli_sdk/answers/route_answer.py,sha256=JaFQ01oWIVXnsCTCpspsQzRp-1w-GnPosjjtqUvlmw8,928
|
|
16
|
+
cli_sdk/answers/set_answer.py,sha256=EAWSuYE-4F34DxUR8H6zU6q6UBOeIWxFTtGGWp19F1I,1633
|
|
17
|
+
cli_sdk/backends/__init__.py,sha256=tm7kl9d7mR-YTFf4Tc90FjqdVu9JUbYce7Hjfo6LW74,1007
|
|
18
|
+
cli_sdk/backends/anthropic.py,sha256=wta0lKxqwhHwbS0IKrfMde4Rs2U7Tbn2_43_QwmbtkM,734
|
|
19
|
+
cli_sdk/backends/azure_openai.py,sha256=xskNICPOUT1ztNi0GgPIKP7ncGHzP6LUsgHymWZVLM8,699
|
|
20
|
+
cli_sdk/backends/base.py,sha256=nTJL2mDGotsHv4SuoyAs4bzodSecOMgDTsNpqNXmPtk,8541
|
|
21
|
+
cli_sdk/backends/bedrock.py,sha256=mmu8VgcVf7FFPtF3tZqpwgJmyMCaGP7ioCDn2nttG1k,544
|
|
22
|
+
cli_sdk/backends/custom.py,sha256=eTqxQ8NQHTgGBQYhlKA7UHyIOt5HBowzNt8bt1un2lY,179
|
|
23
|
+
cli_sdk/backends/gemini.py,sha256=qtIb7vq9QsANd8iF3ETaDpTjAec1NRYCnEdqgXWYbZk,483
|
|
24
|
+
cli_sdk/backends/openai.py,sha256=XPnO40JEMCyt1SmsRVS1VKZFJ9FKZykycornPdFp2Ts,760
|
|
25
|
+
cli_sdk/backends/openrouter.py,sha256=HjrZO9Lg826Kc-0WlXvFgraA7LYIrmZ4xE9CG6V6izY,735
|
|
26
|
+
cli_sdk/backends/sglang.py,sha256=GhJ7hKuZ4kum4kHC5WprXtoeM5cPqTFhFR97zE640bU,1023
|
|
27
|
+
cli_sdk/backends/vllm.py,sha256=Iw9ZP5DBllySMQevu80ejcUglaVG3UQFukuOHDX6TeU,1061
|
|
28
|
+
cli_sdk/calibration/__init__.py,sha256=-xgLiKg4Qr4O9NHT_3_6pPYW266qSO_47efVINnKzJU,832
|
|
29
|
+
cli_sdk/calibration/audit.py,sha256=xdEMnueB4Iwv9kP0dRezDoTu7nMkskYCSJDVOxtan8o,3266
|
|
30
|
+
cli_sdk/calibration/examples.py,sha256=kOHgf8nQxUJbXEEzzMLmdN0R69UXMOoG-TQp2MbMvvc,2066
|
|
31
|
+
cli_sdk/calibration/label_efficient.py,sha256=06B9XRgXvjhM06mfH8ZqxS-Sl68Mcj4y10fO9hFmd0w,3370
|
|
32
|
+
cli_sdk/calibration/profile.py,sha256=du5QWsHkFLgE8N4yccgvgOGVxKJsDZ7GkSvKT--sJDU,9305
|
|
33
|
+
cli_sdk/client/__init__.py,sha256=3F-eSatsWf9RN93nOHXbMxhzDNkgkFvpG1dBFmkbKD4,267
|
|
34
|
+
cli_sdk/client/async_client.py,sha256=FVxsXDIDlNbqFCIk28aIF-antj5cGtvYeCsx8c42YEM,4470
|
|
35
|
+
cli_sdk/client/base.py,sha256=f4z5gSkzqcmcK8CZCx5obOeiDjD1q76YCxvs_Et0F8Q,8352
|
|
36
|
+
cli_sdk/client/retries.py,sha256=BxgH0-MmUgqV9_7LiSyYKm14W-tiLBdxsNPj3uM3ERE,2824
|
|
37
|
+
cli_sdk/client/sync_client.py,sha256=Jx0z3LCeYoSKuhazsK2wMF1s61i8lLu-Cw-I_5MD_00,5445
|
|
38
|
+
cli_sdk/evidence/__init__.py,sha256=pLbfnSp0CZDnr-kJIR5bzQ6sfq3NDCjIafY0irSMFxk,2115
|
|
39
|
+
cli_sdk/evidence/_prompts.py,sha256=SiZJQv4yAXg65UgaXXekwdOlgVvANlXHyD1wzHuPKIk,6758
|
|
40
|
+
cli_sdk/evidence/anthropic.py,sha256=3Y1WblGJmGbaCOTyRKbKxgBs_OXgN8f12RmTQCJv6c4,6221
|
|
41
|
+
cli_sdk/evidence/base.py,sha256=EIleGWAllfLB-tjNELQ944EXIaYy9TVFjtvi4zntB3w,5506
|
|
42
|
+
cli_sdk/evidence/langchain.py,sha256=2MfVqDNqSVaBzP4NDnmox8diSNJkweO_LShovmxpsS0,5061
|
|
43
|
+
cli_sdk/evidence/mock.py,sha256=EsEivEQFXVsV-jDIEIlRgbNuU540j7p4k3ZFh60HdOQ,5977
|
|
44
|
+
cli_sdk/evidence/openai_compatible.py,sha256=2-mHBZv8DL5ileSiuOkOCeQUDi4zXptxkP1LRPETCJQ,11790
|
|
45
|
+
cli_sdk/integrations/__init__.py,sha256=P5--yExihe4npc0OWukB27qdwz7NSg__eGhhgHOxNnw,1410
|
|
46
|
+
cli_sdk/integrations/_guard.py,sha256=FsyPIZTZ-tQTYeJFF5lMV4YBNIaRetXU6BrlLQwPtZc,16767
|
|
47
|
+
cli_sdk/integrations/agent_framework.py,sha256=tZnSqJDnEU5hWSWju6q2fiFd6IWkSiDxoeXeAnNV5sU,6358
|
|
48
|
+
cli_sdk/integrations/google_adk.py,sha256=_ty3U_hCDRvXiYuKFRC7TuFGoK3car5ns7A8_Tsikl8,6891
|
|
49
|
+
cli_sdk/integrations/langchain.py,sha256=cXvpbeA-sfrr0xLr368p03hPpuo2VGHRgN7liHOaoEo,11684
|
|
50
|
+
cli_sdk/integrations/langgraph.py,sha256=iTdKMyMJw7rYJ9nCQaLP2dFeELy3EhuhRunD2TXPsEk,9889
|
|
51
|
+
cli_sdk/local/__init__.py,sha256=_UPZ3--X3SjKW8wGuYOlNJHMKs06ntb1B5xNwjyARUc,696
|
|
52
|
+
cli_sdk/local/client.py,sha256=r7IX48IKsmdahQlxr1c7o60UQmVe6zwsTVLDEpmrRSc,28139
|
|
53
|
+
cli_sdk/local/engine.py,sha256=bPgPQrBeOqvWBPpCTQn4D_pliXUlcUaYHSX76sLhsFM,29644
|
|
54
|
+
cli_sdk/local/scoring.py,sha256=8XWAP-sFNsZMqCpc8CuP2ldOYIzaro0bGiLzHE6Z44I,6698
|
|
55
|
+
cli_sdk/local/store.py,sha256=2TjZSdHzqjnw1swA90YdVaOan2TTOUfdxNTXtC3Lvo8,5304
|
|
56
|
+
cli_sdk/monitoring/__init__.py,sha256=jqc85muXbGlV9FtX0xJMW9HWze-u0ailhCwKQ8cVuak,297
|
|
57
|
+
cli_sdk/monitoring/alerts.py,sha256=__FSsV4S7r2SNQcvnZWAtkTECHwYfnRaPlhHem54gHQ,1637
|
|
58
|
+
cli_sdk/monitoring/monitor.py,sha256=EB-Z8IKw_hqTRg9g58rDf4F-3ZckRYu5vMnrgqwYbFA,7047
|
|
59
|
+
cli_sdk/queries/__init__.py,sha256=CXOXWGcL346ax4ReaWaX-kvx_vEEolgff4pw5P2AiDQ,529
|
|
60
|
+
cli_sdk/queries/base.py,sha256=lr9lUme_cRmLRBOTm36WevcPuTrYEaVZf9CpKU58C9c,2180
|
|
61
|
+
cli_sdk/queries/belief.py,sha256=DFy3dstKONZ-9ocGIRZaqNKhWwdkEzVoVtNtAsn5lVY,806
|
|
62
|
+
cli_sdk/queries/claim.py,sha256=MeZfUWCecndI9bqf70CJF8UqM96reK9SxdL297ErZrg,893
|
|
63
|
+
cli_sdk/queries/gate.py,sha256=QZhZimscShYbxr3szgTRhbqqUD-I7H4E0yCU1uhwrRE,1391
|
|
64
|
+
cli_sdk/queries/interval.py,sha256=LADAdWOGryQIMiy5EZJEhQdrnMS4E840uWxT4V0ba4I,1280
|
|
65
|
+
cli_sdk/queries/judge.py,sha256=s1rd9goNKu49uiziOF_LRU9ktkmJAD-jCVinChjrpTk,922
|
|
66
|
+
cli_sdk/queries/route.py,sha256=qk9bFk5nSoSMrEbR-rkCHhfnK3geMbt4MllfJsSKXQo,1423
|
|
67
|
+
cli_sdk/queries/set.py,sha256=fooAqL8QxcIdlua_pksllZhK36tHeDLuY-8RejB-juk,1336
|
|
68
|
+
cli_sdk/stats/__init__.py,sha256=-bxfJECa684Btsid-YEWXVA_r8aHf38zXHZauHYAQ4s,1090
|
|
69
|
+
cli_sdk/stats/conformal/__init__.py,sha256=yEy_7Gdnne9NsLPistAMNkLFticZq2bgtGbUuTy_mzU,767
|
|
70
|
+
cli_sdk/stats/conformal/_quantile.py,sha256=p2lyXus7J_z-MQg-K0HDS5l5Xy6v6SjeNjgqkZSQuFw,4929
|
|
71
|
+
cli_sdk/stats/conformal/aps.py,sha256=HggLTQZ98TJQsNmvWo8KhTlUD9iZLDKzAZIRV8d7qh8,3769
|
|
72
|
+
cli_sdk/stats/conformal/cqr.py,sha256=Jbxml3-32kj3BK-31QA8q2Xip9lWoIcIiLmQwAkedDg,1564
|
|
73
|
+
cli_sdk/stats/conformal/crc.py,sha256=bEw3pp_hBvUvhX2-0M3tJOZAio58__UWzt8J5uA8npE,2710
|
|
74
|
+
cli_sdk/stats/conformal/lac.py,sha256=1LLReheW0ec729xzIhhC1TVouts44Ms6ciOXB9rp6G4,1341
|
|
75
|
+
cli_sdk/stats/conformal/ltt.py,sha256=Wk9I7KVxH_elrKrAFAodegXHjmLCAfebFzonf_37-DQ,2906
|
|
76
|
+
cli_sdk/stats/conformal/mondrian.py,sha256=CZDzsJrJZjctMBDeh34rPgBiM1YSGbZ_Do6-UFOhf0I,2738
|
|
77
|
+
cli_sdk/stats/conformal/raps.py,sha256=IahBUpDtIuh5R4DJf9A6w23-TSBsEBuxYXMNXhH-T3I,3250
|
|
78
|
+
cli_sdk/stats/conformal/rcps.py,sha256=2V9BZAWBCqY7SkOlePwln1rtrCfpUFBOZm6Zx6iPJ_I,3743
|
|
79
|
+
cli_sdk/stats/evalues/__init__.py,sha256=Y0gNFZt124NenfuRrm1aRFsTez6JEh63_diN0NSkbmo,747
|
|
80
|
+
cli_sdk/stats/evalues/anytime.py,sha256=iUcwAcBOMIR6yTp7Gg5rzb4Rg5ka-UnePSJAIbnXIJk,5147
|
|
81
|
+
cli_sdk/stats/evalues/ebh.py,sha256=VCStp3_1iSvU8IKnd7n1VO9SGk0krGz871eITsX-7i0,1935
|
|
82
|
+
cli_sdk/stats/evalues/eprocess.py,sha256=KhoPbKGxvqq5B58THSJPF93p3WBdTS5sdisOuAasjx0,2231
|
|
83
|
+
cli_sdk/stats/evalues/ppi.py,sha256=bBZLC1B6Ns3Vm0AiiUVDocnrEYtGg34ZwfCHsaRqvjg,3222
|
|
84
|
+
cli_sdk/stats/venn_abers/__init__.py,sha256=Ho1UEkOyNR94lF5vOIEM6b1p3oTYo1ukvViZfgAsk0A,463
|
|
85
|
+
cli_sdk/stats/venn_abers/_isotonic.py,sha256=ux3r0_CcjsuRj3hvCie8ZWgHxLd9R_CEueiQ9IhHqls,1473
|
|
86
|
+
cli_sdk/stats/venn_abers/cvap.py,sha256=f3fORbBC0ShnUyYNw747kcHpT4lLvR2fK9-BQ3jQ9TE,2356
|
|
87
|
+
cli_sdk/stats/venn_abers/ivap.py,sha256=Yua1H2OHiDUKw6SMHhEeDmx4sIsGzk40_wkvwZCGyNU,4650
|
|
88
|
+
cci_sdk-0.1.0.dist-info/METADATA,sha256=t5TKvtlBmZxuGHrrLurlIBXbyJrZL4Gs9Nh0dTcC6zs,16746
|
|
89
|
+
cci_sdk-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
90
|
+
cci_sdk-0.1.0.dist-info/entry_points.txt,sha256=3Habu-N3nF77zkm0IqAWenL3Fuvc300ucebniR74FgY,46
|
|
91
|
+
cci_sdk-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
92
|
+
cci_sdk-0.1.0.dist-info/RECORD,,
|