cci-sdk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. cci_sdk-0.1.0.dist-info/METADATA +430 -0
  2. cci_sdk-0.1.0.dist-info/RECORD +92 -0
  3. cci_sdk-0.1.0.dist-info/WHEEL +4 -0
  4. cci_sdk-0.1.0.dist-info/entry_points.txt +2 -0
  5. cci_sdk-0.1.0.dist-info/licenses/LICENSE +202 -0
  6. cli_sdk/__init__.py +127 -0
  7. cli_sdk/answers/__init__.py +29 -0
  8. cli_sdk/answers/base.py +28 -0
  9. cli_sdk/answers/belief_answer.py +45 -0
  10. cli_sdk/answers/claim_answer.py +53 -0
  11. cli_sdk/answers/gate_answer.py +33 -0
  12. cli_sdk/answers/guarantee.py +163 -0
  13. cli_sdk/answers/interval_answer.py +34 -0
  14. cli_sdk/answers/judge_answer.py +30 -0
  15. cli_sdk/answers/response.py +130 -0
  16. cli_sdk/answers/route_answer.py +29 -0
  17. cli_sdk/answers/set_answer.py +45 -0
  18. cli_sdk/backends/__init__.py +37 -0
  19. cli_sdk/backends/anthropic.py +22 -0
  20. cli_sdk/backends/azure_openai.py +22 -0
  21. cli_sdk/backends/base.py +192 -0
  22. cli_sdk/backends/bedrock.py +20 -0
  23. cli_sdk/backends/custom.py +5 -0
  24. cli_sdk/backends/gemini.py +18 -0
  25. cli_sdk/backends/openai.py +22 -0
  26. cli_sdk/backends/openrouter.py +23 -0
  27. cli_sdk/backends/sglang.py +29 -0
  28. cli_sdk/backends/vllm.py +31 -0
  29. cli_sdk/calibration/__init__.py +28 -0
  30. cli_sdk/calibration/audit.py +94 -0
  31. cli_sdk/calibration/examples.py +55 -0
  32. cli_sdk/calibration/label_efficient.py +78 -0
  33. cli_sdk/calibration/profile.py +236 -0
  34. cli_sdk/cli_tool.py +151 -0
  35. cli_sdk/client/__init__.py +7 -0
  36. cli_sdk/client/async_client.py +113 -0
  37. cli_sdk/client/base.py +207 -0
  38. cli_sdk/client/retries.py +72 -0
  39. cli_sdk/client/sync_client.py +140 -0
  40. cli_sdk/constants.py +31 -0
  41. cli_sdk/evidence/__init__.py +45 -0
  42. cli_sdk/evidence/_prompts.py +165 -0
  43. cli_sdk/evidence/anthropic.py +140 -0
  44. cli_sdk/evidence/base.py +121 -0
  45. cli_sdk/evidence/langchain.py +114 -0
  46. cli_sdk/evidence/mock.py +133 -0
  47. cli_sdk/evidence/openai_compatible.py +278 -0
  48. cli_sdk/exceptions.py +60 -0
  49. cli_sdk/integrations/__init__.py +33 -0
  50. cli_sdk/integrations/_guard.py +356 -0
  51. cli_sdk/integrations/agent_framework.py +140 -0
  52. cli_sdk/integrations/google_adk.py +148 -0
  53. cli_sdk/integrations/langchain.py +251 -0
  54. cli_sdk/integrations/langgraph.py +216 -0
  55. cli_sdk/local/__init__.py +14 -0
  56. cli_sdk/local/client.py +532 -0
  57. cli_sdk/local/engine.py +645 -0
  58. cli_sdk/local/scoring.py +170 -0
  59. cli_sdk/local/store.py +151 -0
  60. cli_sdk/monitoring/__init__.py +6 -0
  61. cli_sdk/monitoring/alerts.py +47 -0
  62. cli_sdk/monitoring/monitor.py +180 -0
  63. cli_sdk/py.typed +0 -0
  64. cli_sdk/queries/__init__.py +15 -0
  65. cli_sdk/queries/base.py +59 -0
  66. cli_sdk/queries/belief.py +25 -0
  67. cli_sdk/queries/claim.py +27 -0
  68. cli_sdk/queries/gate.py +38 -0
  69. cli_sdk/queries/interval.py +33 -0
  70. cli_sdk/queries/judge.py +27 -0
  71. cli_sdk/queries/route.py +35 -0
  72. cli_sdk/queries/set.py +35 -0
  73. cli_sdk/stats/__init__.py +25 -0
  74. cli_sdk/stats/conformal/__init__.py +28 -0
  75. cli_sdk/stats/conformal/_quantile.py +144 -0
  76. cli_sdk/stats/conformal/aps.py +98 -0
  77. cli_sdk/stats/conformal/cqr.py +42 -0
  78. cli_sdk/stats/conformal/crc.py +71 -0
  79. cli_sdk/stats/conformal/lac.py +37 -0
  80. cli_sdk/stats/conformal/ltt.py +77 -0
  81. cli_sdk/stats/conformal/mondrian.py +72 -0
  82. cli_sdk/stats/conformal/raps.py +85 -0
  83. cli_sdk/stats/conformal/rcps.py +103 -0
  84. cli_sdk/stats/evalues/__init__.py +20 -0
  85. cli_sdk/stats/evalues/anytime.py +124 -0
  86. cli_sdk/stats/evalues/ebh.py +53 -0
  87. cli_sdk/stats/evalues/eprocess.py +55 -0
  88. cli_sdk/stats/evalues/ppi.py +91 -0
  89. cli_sdk/stats/venn_abers/__init__.py +11 -0
  90. cli_sdk/stats/venn_abers/_isotonic.py +39 -0
  91. cli_sdk/stats/venn_abers/cvap.py +62 -0
  92. cli_sdk/stats/venn_abers/ivap.py +104 -0
@@ -0,0 +1,430 @@
1
+ Metadata-Version: 2.5
2
+ Name: cci-sdk
3
+ Version: 0.1.0
4
+ Summary: Python SDK and standalone statistics engine for Conformal Logit Inference (CLI) — conformal prediction, Venn-Abers calibration, and e-values for LLM decisions.
5
+ Project-URL: Homepage, https://cci.gitdate.ink
6
+ Project-URL: Documentation, https://cci-docs.gitdate.ink
7
+ Project-URL: Repository, https://github.com/rahvis/cci-sdk
8
+ Author-email: CCI <rahul.vishwakarma@workonward.org>
9
+ License: Apache-2.0
10
+ License-File: LICENSE
11
+ Keywords: calibration,conformal-prediction,e-values,llm,uncertainty-quantification,venn-abers
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: Apache Software License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Requires-Python: >=3.9
18
+ Requires-Dist: httpx>=0.27
19
+ Requires-Dist: numpy>=1.24
20
+ Provides-Extra: agent-framework
21
+ Requires-Dist: agent-framework-core>=1.19; extra == 'agent-framework'
22
+ Provides-Extra: all
23
+ Requires-Dist: agent-framework-core>=1.19; extra == 'all'
24
+ Requires-Dist: anthropic>=1.0; extra == 'all'
25
+ Requires-Dist: google-adk>=2.0; extra == 'all'
26
+ Requires-Dist: langchain-core>=1.0; extra == 'all'
27
+ Requires-Dist: langchain>=1.0; extra == 'all'
28
+ Requires-Dist: langgraph>=1.0; extra == 'all'
29
+ Requires-Dist: openai>=1.40; extra == 'all'
30
+ Provides-Extra: anthropic
31
+ Requires-Dist: anthropic>=1.0; extra == 'anthropic'
32
+ Provides-Extra: dev
33
+ Requires-Dist: mypy>=1.10; extra == 'dev'
34
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
35
+ Requires-Dist: pytest>=8.0; extra == 'dev'
36
+ Requires-Dist: ruff>=0.5; extra == 'dev'
37
+ Provides-Extra: google-adk
38
+ Requires-Dist: google-adk>=2.0; extra == 'google-adk'
39
+ Provides-Extra: langchain
40
+ Requires-Dist: langchain-core>=1.0; extra == 'langchain'
41
+ Requires-Dist: langchain>=1.0; extra == 'langchain'
42
+ Requires-Dist: langgraph>=1.0; extra == 'langchain'
43
+ Provides-Extra: langgraph
44
+ Requires-Dist: langchain-core>=1.0; extra == 'langgraph'
45
+ Requires-Dist: langgraph>=1.0; extra == 'langgraph'
46
+ Provides-Extra: openai
47
+ Requires-Dist: openai>=1.40; extra == 'openai'
48
+ Description-Content-Type: text/markdown
49
+
50
+ # cci-sdk — Python SDK for Conformal Logit Inference
51
+
52
+ Conformal Logit Inference (CLI) turns the raw output of any LLM into
53
+ finite-sample statistical guarantees. This package contains:
54
+
55
+ - **`cli_sdk`** — sync and async clients for the hosted API, the seven
56
+ guarantee-bearing query primitives, typed answers, calibration profiles,
57
+ and drift monitors.
58
+ - **`cli_sdk.stats`** — the standalone statistics engine (conformal
59
+ prediction, Venn-Abers calibration, e-values). It needs only NumPy and
60
+ makes no network calls, so it runs inside air-gapped environments on
61
+ precomputed score arrays.
62
+
63
+ - **`cli_sdk.local`** and **`cli_sdk.evidence`** — local mode: calibrate and
64
+ evaluate the same queries in-process on your own model (OpenAI, Azure
65
+ OpenAI, Anthropic, Gemini, or open weights such as Gemma on vLLM or
66
+ SGLang), with calibration profiles stored as reviewable JSON files.
67
+ - **`cli_sdk.integrations`** — calibrated guardrails for agent tool calls in
68
+ LangChain, LangGraph, Google ADK and Microsoft Agent Framework: allow,
69
+ escalate to a human, or block, each with a stated guarantee.
70
+
71
+ Every guarantee is marginal or group-conditional over data exchangeable with
72
+ a stated calibration set. None is a promise about one decision in isolation.
73
+
74
+ ## Install
75
+
76
+ ```bash
77
+ pip install cci-sdk # or: uv add cli-sdk
78
+ ```
79
+
80
+ Optional extras for model providers and agent frameworks (Python 3.10+ for
81
+ the frameworks):
82
+
83
+ ```bash
84
+ pip install "cci-sdk[openai]" # OpenAI, Azure OpenAI, vLLM, SGLang evidence backends
85
+ pip install "cci-sdk[anthropic]" # Claude evidence backend
86
+ pip install "cci-sdk[langchain,openai]" # or [langgraph], [google-adk], [agent-framework], [all]
87
+ ```
88
+
89
+ For development in this repository:
90
+
91
+ ```bash
92
+ cd sdk/python
93
+ pip install -e ".[dev]"
94
+ pytest
95
+ ```
96
+
97
+ ## Authenticate
98
+
99
+ ```bash
100
+ export CLI_API_KEY=sk_live_...
101
+ # optional, for a self-hosted deployment:
102
+ export CLI_BASE_URL=https://cli.internal.example.com/v1
103
+ ```
104
+
105
+ A key is not required when `CLI_BASE_URL` points at `localhost`.
106
+
107
+ ## Accessing the API
108
+
109
+ ### Over HTTP
110
+
111
+ Every call is `POST https://cci.gitdate.ink/api/v1/evaluate` with a bearer token:
112
+
113
+ ```bash
114
+ curl https://cci.gitdate.ink/api/v1/evaluate \
115
+ -H "Authorization: Bearer $CLI_API_KEY" \
116
+ -H "Content-Type: application/json" \
117
+ -d '{
118
+ "context": {"ticket": "My payouts have been failing for 3 days."},
119
+ "backend": {"provider": "openai", "model": "gpt-4.1-2025-04-14"},
120
+ "queries": {
121
+ "department": {
122
+ "type": "set",
123
+ "instructions": "Which team should handle this ticket?",
124
+ "options": {"billing": null, "technical": null, "sales": null},
125
+ "calibration_profile": "support-routing-v3",
126
+ "alpha": 0.10
127
+ }
128
+ }
129
+ }'
130
+ ```
131
+
132
+ | Endpoint | Purpose |
133
+ |---|---|
134
+ | `POST /v1/evaluate` | Evaluate a context against one or more queries |
135
+ | `POST /v1/calibration-profiles` | Create a calibration profile |
136
+ | `GET /v1/calibration-profiles/{name}` | Size, coverage interval, audit history |
137
+ | `POST /v1/calibration-profiles/{name}/examples` | Add labelled examples |
138
+ | `POST /v1/calibration-profiles/{name}/label-with-judge` | Judge-labelled pool plus a human sample, combined validly |
139
+ | `POST /v1/calibration-profiles/{name}/audit` | Audit against fresh labelled examples |
140
+ | `POST /v1/calibration-profiles/{name}/monitors` | Create an anytime-valid drift monitor |
141
+ | `GET /v1/calibration-profiles/{name}/monitors/{id}/alerts` | Poll monitor alerts |
142
+ | `GET /v1/backends` | Configured backends and their detected access level |
143
+
144
+ ### With the SDK
145
+
146
+ ```python
147
+ from cli_sdk import CLIClient, Set, Gate
148
+
149
+ with CLIClient() as client:
150
+ result = client.evaluate(
151
+ context={"ticket": "My payouts have been failing for 3 days."},
152
+ backend={"provider": "openai", "model": "gpt-4.1-2025-04-14"},
153
+ queries={
154
+ "department": Set(
155
+ instructions="Which team should handle this ticket?",
156
+ options={"billing": None, "technical": None, "sales": None},
157
+ calibration_profile="support-routing-v3",
158
+ alpha=0.10,
159
+ ),
160
+ "route": Gate(
161
+ instructions="Auto-route this ticket without human review?",
162
+ calibration_profile="support-routing-v3",
163
+ guarantee="fdr",
164
+ target=0.05,
165
+ ),
166
+ },
167
+ )
168
+
169
+ department = result.answers["department"]
170
+ print(department.set, department.guarantee.describe())
171
+
172
+ if result.answers["route"].approved:
173
+ assign_queue(department.top)
174
+ else:
175
+ send_to_human_triage(department.set)
176
+ ```
177
+
178
+ The async client has the same interface, with `await` on each call (`monitors.poll` returns a list rather than a generator):
179
+
180
+ ```python
181
+ from cli_sdk import AsyncCLIClient
182
+
183
+ async with AsyncCLIClient() as client:
184
+ result = await client.evaluate(context=..., backend=..., queries=...)
185
+ ```
186
+
187
+ ## The seven primitives
188
+
189
+ | Primitive | Returns | Guarantee |
190
+ |---|---|---|
191
+ | `Belief` | Venn-Abers interval `[p0, p1]` for one statement | calibrated probability bracket |
192
+ | `Set` | a set of options | coverage >= 1 - alpha |
193
+ | `Interval` | a conformalized ordinal/continuous interval | coverage >= 1 - alpha |
194
+ | `Gate` | `auto_approve` / `escalate` / `abstain` | risk <= alpha, risk with probability 1 - delta, or batch FDR <= q |
195
+ | `Claim` | long-form output filtered to supported claims | P(all retained claims true) >= 1 - alpha |
196
+ | `Judge` | a verdict with an escalation cascade | human agreement >= 1 - alpha on accepted verdicts |
197
+ | `Route` | which backend in a cascade served the request | cost or accuracy bound |
198
+
199
+ Every answer carries `answer.guarantee`, a `Guarantee` with `type`,
200
+ `method`, `alpha`/`delta`/`target`, `calibration_profile`, `calibration_n`,
201
+ `coverage_ci`, and `last_audited`. `guarantee.is_heuristic` is `True` when an
202
+ answer carries no formal guarantee, for example because its profile is still
203
+ below the minimum size. Pass `strict_guarantees=True` to the client to raise
204
+ `InsufficientCalibrationError` instead of receiving heuristic answers.
205
+
206
+ ## Typed responses
207
+
208
+ ```python
209
+ from cli_sdk import EvaluateResponse, SetAnswer
210
+
211
+ class RoutingResponse(EvaluateResponse):
212
+ department: SetAnswer
213
+
214
+ result = client.evaluate(..., response_model=RoutingResponse)
215
+ result.department # a SetAnswer; a missing or mistyped answer raises at parse time
216
+ ```
217
+
218
+ ## Calibration profiles
219
+
220
+ ```python
221
+ client.calibration_profiles.create(
222
+ name="support-routing-v3",
223
+ backend={"provider": "openai", "model": "gpt-4.1-2025-04-14"},
224
+ method="APS",
225
+ alpha=0.10,
226
+ group_by="account_tier", # optional Mondrian (per-group) calibration
227
+ )
228
+ client.calibration_profiles.add_examples(
229
+ "support-routing-v3",
230
+ examples=[{"context": {"ticket": "I was charged twice."}, "label": "billing"}],
231
+ )
232
+ profile = client.calibration_profiles.get("support-routing-v3")
233
+ profile.n, profile.minimum_n, profile.recommended_n, profile.realized_coverage_ci
234
+ ```
235
+
236
+ | alpha | hard minimum n | recommended n |
237
+ |---|---|---|
238
+ | 0.20 | 4 | ~300 |
239
+ | 0.10 | 9 | ~1,000 |
240
+ | 0.05 | 19 | ~1,500 |
241
+ | 0.01 | 99 | ~2,500 |
242
+
243
+ Label-efficient calibration labels a pool with a judge and a random human
244
+ subset, combined so the result is valid regardless of judge quality:
245
+
246
+ ```python
247
+ client.calibration_profiles.label_with_judge(
248
+ "support-routing-v3",
249
+ judge={"provider": "openai", "model": "gpt-4.1-2025-04-14"},
250
+ unlabelled_examples=pool,
251
+ human_labelled_sample_size=300,
252
+ )
253
+ ```
254
+
255
+ ## Drift monitoring
256
+
257
+ ```python
258
+ client.calibration_profiles.monitors.create(
259
+ "support-routing-v3", type="coverage", target=0.90,
260
+ false_alarm_rate=0.05, labelled_sample_rate=0.02,
261
+ )
262
+ for alert in client.calibration_profiles.monitors.poll("support-routing-v3"):
263
+ pause_auto_actioning()
264
+ ```
265
+
266
+ Monitors are e-processes: the false-alarm rate holds no matter how often or
267
+ for how long you check them. `LocalMonitor` runs the same test in-process:
268
+
269
+ ```python
270
+ from cli_sdk import LocalMonitor
271
+
272
+ monitor = LocalMonitor(type="coverage", target=0.90, false_alarm_rate=0.05)
273
+ for label, prediction_set in stream:
274
+ if alert := monitor.update(label in prediction_set):
275
+ page_oncall(alert)
276
+ ```
277
+
278
+ ## Backends
279
+
280
+ ```python
281
+ from cli_sdk import OpenAIBackend, AnthropicBackend, VLLMBackend
282
+
283
+ OpenAIBackend(model="gpt-4.1-2025-04-14") # L1 on non-reasoning configs
284
+ AnthropicBackend(model="claude-sonnet-5", sample_count=20) # L0: sampling only
285
+ VLLMBackend(model="Qwen/Qwen3-8B", base_url="http://vllm:8000") # up to L4
286
+ ```
287
+
288
+ Plain dicts (`{"provider": "openai", "model": ...}`) work anywhere a backend
289
+ is accepted.
290
+
291
+ ### Bring your own model
292
+
293
+ A `CustomBackend` runs your model locally and sends only the evidence
294
+ (probabilities, samples, scores) to CLI. Implement the methods your access
295
+ level supports:
296
+
297
+ ```python
298
+ from cli_sdk import CLIClient, CustomBackend
299
+
300
+ class MyEngine(CustomBackend):
301
+ access_level = "L1"
302
+
303
+ def score_options(self, context, instructions, options):
304
+ return my_model.option_probabilities(context, instructions, list(options))
305
+
306
+ def sample(self, context, instructions, n):
307
+ return [my_model.generate(context, instructions) for _ in range(n)]
308
+
309
+ client = CLIClient(backend=MyEngine())
310
+ ```
311
+
312
+ ## Local mode
313
+
314
+ `LocalCLIClient` runs the same queries fully in-process: it scores labelled
315
+ examples with a model you bring, stores the calibration profile as a JSON
316
+ file, and answers with the same typed answers and guarantee cards as the
317
+ hosted client. Nothing leaves your process except calls to your own model
318
+ provider.
319
+
320
+ ```python
321
+ from cli_sdk import Gate, LocalCLIClient
322
+ from cli_sdk.evidence import OpenAIEvidenceBackend, vllm_backend
323
+
324
+ evidence = OpenAIEvidenceBackend("gpt-4.1-mini", api_key="YOUR_OPENAI_API_KEY")
325
+ # or open weights: evidence = vllm_backend("google/gemma-4-12B-it", base_url="http://localhost:8000/v1")
326
+
327
+ client = LocalCLIClient(evidence, store=".cli_profiles")
328
+ refund = Gate(instructions="Is issuing this refund correct under the policy?",
329
+ calibration_profile="refund-approvals-v1", guarantee="risk", target=0.05)
330
+ client.calibrate(refund, labelled_examples) # [{"context": {...}, "label": True}, ...]
331
+ answer = client.evaluate(context=case, queries={"refund": refund}).answers["refund"]
332
+ print(answer.decision, answer.guarantee.describe())
333
+ ```
334
+
335
+ Profiles are fingerprinted with the backend configuration and the query's
336
+ prompt: change either and answers fall back to their safe heuristic form
337
+ (escalate, every option, `[0, 1]`) until you recalibrate. See the
338
+ [local mode guide](https://cci-docs.gitdate.ink/agents/local-mode).
339
+
340
+ ## Agent frameworks
341
+
342
+ `ToolGuard` decides each proposed tool call from a calibrated answer and
343
+ maps it onto the framework's native human-in-the-loop mechanism:
344
+
345
+ ```python
346
+ from cli_sdk.integrations import GuardRule, ToolGuard
347
+ from cli_sdk.integrations.langchain import cli_middleware
348
+
349
+ guard = ToolGuard(client, [GuardRule(tool="issue_refund", query=refund, context=refund_context)])
350
+ agent = create_agent(model, tools=[lookup_order, issue_refund],
351
+ middleware=cli_middleware(guard), checkpointer=InMemorySaver())
352
+ ```
353
+
354
+ | Framework | Module | Hook | Human review |
355
+ |---|---|---|---|
356
+ | LangChain 1.x | `cli_sdk.integrations.langchain` | `wrap_tool_call` middleware | `HumanInTheLoopMiddleware` |
357
+ | LangGraph | `cli_sdk.integrations.langgraph` | guard node before `ToolNode` | `interrupt()` / `Command(resume=...)` |
358
+ | Google ADK 2.x | `cli_sdk.integrations.google_adk` | `before_tool_callback` or plugin | tool confirmation |
359
+ | Microsoft Agent Framework | `cli_sdk.integrations.agent_framework` | `FunctionMiddleware` | function approval requests |
360
+
361
+ Eight runnable examples in risk-sensitive domains (refunds, patient triage,
362
+ insurance claims, discharge summaries, AML holds, trial screening, credit
363
+ tiers, drug safety) are in `examples/agents/`. Each runs offline with
364
+ `--provider mock` and with your own keys for OpenAI, Azure, Anthropic,
365
+ Gemini, or Gemma on vLLM or SGLang. See the
366
+ [agent frameworks guide](https://cci-docs.gitdate.ink/agents).
367
+
368
+ ## Offline statistics engine
369
+
370
+ ```python
371
+ from cli_sdk.stats.conformal import aps, crc
372
+ from cli_sdk.stats.venn_abers import ivap
373
+ from cli_sdk.stats.evalues import ebh, ppi, CoverageMonitor
374
+
375
+ q_hat = aps.calibrate(cal_probs, cal_labels, alpha=0.10)
376
+ sets = aps.predict(test_probs, q_hat)
377
+
378
+ p0, p1 = ivap.calibrate_and_predict(cal_scores, cal_labels, test_scores)
379
+
380
+ selected = ebh.select(e_values, q=0.10)
381
+ ```
382
+
383
+ | Module | Methods |
384
+ |---|---|
385
+ | `stats.conformal` | LAC, APS, RAPS, CQR, CRC, RCPS, Learn-then-Test, Mondrian |
386
+ | `stats.venn_abers` | IVAP, CVAP, interval merging and width |
387
+ | `stats.evalues` | anytime-valid risk and coverage monitors, e-BH, prediction-powered inference |
388
+
389
+ ## Command-line tool
390
+
391
+ ```bash
392
+ cli calibration show support-routing-v3
393
+ cli calibration audit support-routing-v3 --examples fresh.jsonl --fail-below 0.88
394
+ cli calibration audit-local --covered outcomes.txt --target 0.90
395
+ ```
396
+
397
+ `audit` exits with status 1 on failure, so it can block a deploy in CI.
398
+
399
+ ## Errors
400
+
401
+ | Exception | When |
402
+ |---|---|
403
+ | `AuthenticationError` | 401: missing or invalid API key |
404
+ | `ValidationError` | 422: the request failed validation |
405
+ | `InsufficientCalibrationError` | a profile is too small for the guarantee (raised only with `strict_guarantees=True`) |
406
+ | `RateLimitError` | 429 after retries are exhausted |
407
+ | `BackendError` | 502 / 529: the model backend errored or reported a lower access level than required |
408
+ | `ConfigurationError` | client-side misconfiguration, raised before any request |
409
+
410
+ `429`, `502` and `529` are retried with full-jitter exponential backoff,
411
+ honoring `retry-after`. Configure with `CLIClient(retry=RetryConfig(max_attempts=5, base_delay=0.5))`.
412
+
413
+ ## Package layout
414
+
415
+ ```
416
+ src/cli_sdk/
417
+ client/ sync + async clients, retry policy
418
+ queries/ Belief, Set, Interval, Gate, Claim, Judge, Route
419
+ answers/ typed answers, the Guarantee card, EvaluateResponse
420
+ calibration/ profiles, examples, audits, label-efficient calibration
421
+ monitoring/ hosted monitors, alerts, LocalMonitor
422
+ backends/ one adapter per provider, plus CustomBackend
423
+ stats/ offline engine: conformal/, venn_abers/, evalues/
424
+ local/ LocalCLIClient: in-process calibration and evaluation
425
+ evidence/ evidence backends: OpenAI, Azure, Anthropic, Gemini, vLLM, SGLang, mock
426
+ integrations/ ToolGuard and adapters for LangChain, LangGraph, ADK, Agent Framework
427
+ cli_tool.py the `cli` command
428
+ examples/ runnable end-to-end examples (agents/: agent-framework examples)
429
+ tests/ engine and client tests
430
+ ```
@@ -0,0 +1,92 @@
1
+ cli_sdk/__init__.py,sha256=NKRmlmMkj-qMqknHeJFnc9t8EdiDvn6zoKY6MdNlS4k,2988
2
+ cli_sdk/cli_tool.py,sha256=uE36rN-Y4wHu8rffXlNvX2MGav6I6fgPuEIX9ROUM6M,5849
3
+ cli_sdk/constants.py,sha256=86Of269FcxCAddaeotp-oBJXZ3bg6hnQpsXvYYFY1Ok,1048
4
+ cli_sdk/exceptions.py,sha256=kwGHhJ3sbt3E2_tVBC7DTLRP5i_x1p3nLXGXsJOqD3w,2002
5
+ cli_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ cli_sdk/answers/__init__.py,sha256=ZJ637QK1TEClnoSrcJgUOiMyLGImxvOlSvkDLgZrxO0,910
7
+ cli_sdk/answers/base.py,sha256=QtQ4SRehQY596dSn6SVWBCS4OmjA48jwQxbemZnACKM,634
8
+ cli_sdk/answers/belief_answer.py,sha256=M0RcRB_ZMDwHR9-B_phzcJ4NP10ZNDngR-bOuS1PVgc,1603
9
+ cli_sdk/answers/claim_answer.py,sha256=DKR4GY6dFxFBRCpRUgyyECxzDfQFQYg9rye4ySvosTc,1727
10
+ cli_sdk/answers/gate_answer.py,sha256=YduF85j3jPyfTn3Zx7SjrsNslG9euf5WSbRMV83j4wc,963
11
+ cli_sdk/answers/guarantee.py,sha256=AM5l1Y0IaI1rgZ8KRrIAbsO2wGDHWhqGSlo3F30qtC0,6648
12
+ cli_sdk/answers/interval_answer.py,sha256=8sGmnLtj2W0hm4hM7wbavZQUv9wi87F7RXOn-TlL7pA,1088
13
+ cli_sdk/answers/judge_answer.py,sha256=3muSNv4aQnhJmROkeapR_qKlCSBzWPNykEI9UJiXgS8,894
14
+ cli_sdk/answers/response.py,sha256=h4P9vzrEI2DL5rLQ67LkQKosfvDQGI6cH1L7cOie_vo,4821
15
+ cli_sdk/answers/route_answer.py,sha256=JaFQ01oWIVXnsCTCpspsQzRp-1w-GnPosjjtqUvlmw8,928
16
+ cli_sdk/answers/set_answer.py,sha256=EAWSuYE-4F34DxUR8H6zU6q6UBOeIWxFTtGGWp19F1I,1633
17
+ cli_sdk/backends/__init__.py,sha256=tm7kl9d7mR-YTFf4Tc90FjqdVu9JUbYce7Hjfo6LW74,1007
18
+ cli_sdk/backends/anthropic.py,sha256=wta0lKxqwhHwbS0IKrfMde4Rs2U7Tbn2_43_QwmbtkM,734
19
+ cli_sdk/backends/azure_openai.py,sha256=xskNICPOUT1ztNi0GgPIKP7ncGHzP6LUsgHymWZVLM8,699
20
+ cli_sdk/backends/base.py,sha256=nTJL2mDGotsHv4SuoyAs4bzodSecOMgDTsNpqNXmPtk,8541
21
+ cli_sdk/backends/bedrock.py,sha256=mmu8VgcVf7FFPtF3tZqpwgJmyMCaGP7ioCDn2nttG1k,544
22
+ cli_sdk/backends/custom.py,sha256=eTqxQ8NQHTgGBQYhlKA7UHyIOt5HBowzNt8bt1un2lY,179
23
+ cli_sdk/backends/gemini.py,sha256=qtIb7vq9QsANd8iF3ETaDpTjAec1NRYCnEdqgXWYbZk,483
24
+ cli_sdk/backends/openai.py,sha256=XPnO40JEMCyt1SmsRVS1VKZFJ9FKZykycornPdFp2Ts,760
25
+ cli_sdk/backends/openrouter.py,sha256=HjrZO9Lg826Kc-0WlXvFgraA7LYIrmZ4xE9CG6V6izY,735
26
+ cli_sdk/backends/sglang.py,sha256=GhJ7hKuZ4kum4kHC5WprXtoeM5cPqTFhFR97zE640bU,1023
27
+ cli_sdk/backends/vllm.py,sha256=Iw9ZP5DBllySMQevu80ejcUglaVG3UQFukuOHDX6TeU,1061
28
+ cli_sdk/calibration/__init__.py,sha256=-xgLiKg4Qr4O9NHT_3_6pPYW266qSO_47efVINnKzJU,832
29
+ cli_sdk/calibration/audit.py,sha256=xdEMnueB4Iwv9kP0dRezDoTu7nMkskYCSJDVOxtan8o,3266
30
+ cli_sdk/calibration/examples.py,sha256=kOHgf8nQxUJbXEEzzMLmdN0R69UXMOoG-TQp2MbMvvc,2066
31
+ cli_sdk/calibration/label_efficient.py,sha256=06B9XRgXvjhM06mfH8ZqxS-Sl68Mcj4y10fO9hFmd0w,3370
32
+ cli_sdk/calibration/profile.py,sha256=du5QWsHkFLgE8N4yccgvgOGVxKJsDZ7GkSvKT--sJDU,9305
33
+ cli_sdk/client/__init__.py,sha256=3F-eSatsWf9RN93nOHXbMxhzDNkgkFvpG1dBFmkbKD4,267
34
+ cli_sdk/client/async_client.py,sha256=FVxsXDIDlNbqFCIk28aIF-antj5cGtvYeCsx8c42YEM,4470
35
+ cli_sdk/client/base.py,sha256=f4z5gSkzqcmcK8CZCx5obOeiDjD1q76YCxvs_Et0F8Q,8352
36
+ cli_sdk/client/retries.py,sha256=BxgH0-MmUgqV9_7LiSyYKm14W-tiLBdxsNPj3uM3ERE,2824
37
+ cli_sdk/client/sync_client.py,sha256=Jx0z3LCeYoSKuhazsK2wMF1s61i8lLu-Cw-I_5MD_00,5445
38
+ cli_sdk/evidence/__init__.py,sha256=pLbfnSp0CZDnr-kJIR5bzQ6sfq3NDCjIafY0irSMFxk,2115
39
+ cli_sdk/evidence/_prompts.py,sha256=SiZJQv4yAXg65UgaXXekwdOlgVvANlXHyD1wzHuPKIk,6758
40
+ cli_sdk/evidence/anthropic.py,sha256=3Y1WblGJmGbaCOTyRKbKxgBs_OXgN8f12RmTQCJv6c4,6221
41
+ cli_sdk/evidence/base.py,sha256=EIleGWAllfLB-tjNELQ944EXIaYy9TVFjtvi4zntB3w,5506
42
+ cli_sdk/evidence/langchain.py,sha256=2MfVqDNqSVaBzP4NDnmox8diSNJkweO_LShovmxpsS0,5061
43
+ cli_sdk/evidence/mock.py,sha256=EsEivEQFXVsV-jDIEIlRgbNuU540j7p4k3ZFh60HdOQ,5977
44
+ cli_sdk/evidence/openai_compatible.py,sha256=2-mHBZv8DL5ileSiuOkOCeQUDi4zXptxkP1LRPETCJQ,11790
45
+ cli_sdk/integrations/__init__.py,sha256=P5--yExihe4npc0OWukB27qdwz7NSg__eGhhgHOxNnw,1410
46
+ cli_sdk/integrations/_guard.py,sha256=FsyPIZTZ-tQTYeJFF5lMV4YBNIaRetXU6BrlLQwPtZc,16767
47
+ cli_sdk/integrations/agent_framework.py,sha256=tZnSqJDnEU5hWSWju6q2fiFd6IWkSiDxoeXeAnNV5sU,6358
48
+ cli_sdk/integrations/google_adk.py,sha256=_ty3U_hCDRvXiYuKFRC7TuFGoK3car5ns7A8_Tsikl8,6891
49
+ cli_sdk/integrations/langchain.py,sha256=cXvpbeA-sfrr0xLr368p03hPpuo2VGHRgN7liHOaoEo,11684
50
+ cli_sdk/integrations/langgraph.py,sha256=iTdKMyMJw7rYJ9nCQaLP2dFeELy3EhuhRunD2TXPsEk,9889
51
+ cli_sdk/local/__init__.py,sha256=_UPZ3--X3SjKW8wGuYOlNJHMKs06ntb1B5xNwjyARUc,696
52
+ cli_sdk/local/client.py,sha256=r7IX48IKsmdahQlxr1c7o60UQmVe6zwsTVLDEpmrRSc,28139
53
+ cli_sdk/local/engine.py,sha256=bPgPQrBeOqvWBPpCTQn4D_pliXUlcUaYHSX76sLhsFM,29644
54
+ cli_sdk/local/scoring.py,sha256=8XWAP-sFNsZMqCpc8CuP2ldOYIzaro0bGiLzHE6Z44I,6698
55
+ cli_sdk/local/store.py,sha256=2TjZSdHzqjnw1swA90YdVaOan2TTOUfdxNTXtC3Lvo8,5304
56
+ cli_sdk/monitoring/__init__.py,sha256=jqc85muXbGlV9FtX0xJMW9HWze-u0ailhCwKQ8cVuak,297
57
+ cli_sdk/monitoring/alerts.py,sha256=__FSsV4S7r2SNQcvnZWAtkTECHwYfnRaPlhHem54gHQ,1637
58
+ cli_sdk/monitoring/monitor.py,sha256=EB-Z8IKw_hqTRg9g58rDf4F-3ZckRYu5vMnrgqwYbFA,7047
59
+ cli_sdk/queries/__init__.py,sha256=CXOXWGcL346ax4ReaWaX-kvx_vEEolgff4pw5P2AiDQ,529
60
+ cli_sdk/queries/base.py,sha256=lr9lUme_cRmLRBOTm36WevcPuTrYEaVZf9CpKU58C9c,2180
61
+ cli_sdk/queries/belief.py,sha256=DFy3dstKONZ-9ocGIRZaqNKhWwdkEzVoVtNtAsn5lVY,806
62
+ cli_sdk/queries/claim.py,sha256=MeZfUWCecndI9bqf70CJF8UqM96reK9SxdL297ErZrg,893
63
+ cli_sdk/queries/gate.py,sha256=QZhZimscShYbxr3szgTRhbqqUD-I7H4E0yCU1uhwrRE,1391
64
+ cli_sdk/queries/interval.py,sha256=LADAdWOGryQIMiy5EZJEhQdrnMS4E840uWxT4V0ba4I,1280
65
+ cli_sdk/queries/judge.py,sha256=s1rd9goNKu49uiziOF_LRU9ktkmJAD-jCVinChjrpTk,922
66
+ cli_sdk/queries/route.py,sha256=qk9bFk5nSoSMrEbR-rkCHhfnK3geMbt4MllfJsSKXQo,1423
67
+ cli_sdk/queries/set.py,sha256=fooAqL8QxcIdlua_pksllZhK36tHeDLuY-8RejB-juk,1336
68
+ cli_sdk/stats/__init__.py,sha256=-bxfJECa684Btsid-YEWXVA_r8aHf38zXHZauHYAQ4s,1090
69
+ cli_sdk/stats/conformal/__init__.py,sha256=yEy_7Gdnne9NsLPistAMNkLFticZq2bgtGbUuTy_mzU,767
70
+ cli_sdk/stats/conformal/_quantile.py,sha256=p2lyXus7J_z-MQg-K0HDS5l5Xy6v6SjeNjgqkZSQuFw,4929
71
+ cli_sdk/stats/conformal/aps.py,sha256=HggLTQZ98TJQsNmvWo8KhTlUD9iZLDKzAZIRV8d7qh8,3769
72
+ cli_sdk/stats/conformal/cqr.py,sha256=Jbxml3-32kj3BK-31QA8q2Xip9lWoIcIiLmQwAkedDg,1564
73
+ cli_sdk/stats/conformal/crc.py,sha256=bEw3pp_hBvUvhX2-0M3tJOZAio58__UWzt8J5uA8npE,2710
74
+ cli_sdk/stats/conformal/lac.py,sha256=1LLReheW0ec729xzIhhC1TVouts44Ms6ciOXB9rp6G4,1341
75
+ cli_sdk/stats/conformal/ltt.py,sha256=Wk9I7KVxH_elrKrAFAodegXHjmLCAfebFzonf_37-DQ,2906
76
+ cli_sdk/stats/conformal/mondrian.py,sha256=CZDzsJrJZjctMBDeh34rPgBiM1YSGbZ_Do6-UFOhf0I,2738
77
+ cli_sdk/stats/conformal/raps.py,sha256=IahBUpDtIuh5R4DJf9A6w23-TSBsEBuxYXMNXhH-T3I,3250
78
+ cli_sdk/stats/conformal/rcps.py,sha256=2V9BZAWBCqY7SkOlePwln1rtrCfpUFBOZm6Zx6iPJ_I,3743
79
+ cli_sdk/stats/evalues/__init__.py,sha256=Y0gNFZt124NenfuRrm1aRFsTez6JEh63_diN0NSkbmo,747
80
+ cli_sdk/stats/evalues/anytime.py,sha256=iUcwAcBOMIR6yTp7Gg5rzb4Rg5ka-UnePSJAIbnXIJk,5147
81
+ cli_sdk/stats/evalues/ebh.py,sha256=VCStp3_1iSvU8IKnd7n1VO9SGk0krGz871eITsX-7i0,1935
82
+ cli_sdk/stats/evalues/eprocess.py,sha256=KhoPbKGxvqq5B58THSJPF93p3WBdTS5sdisOuAasjx0,2231
83
+ cli_sdk/stats/evalues/ppi.py,sha256=bBZLC1B6Ns3Vm0AiiUVDocnrEYtGg34ZwfCHsaRqvjg,3222
84
+ cli_sdk/stats/venn_abers/__init__.py,sha256=Ho1UEkOyNR94lF5vOIEM6b1p3oTYo1ukvViZfgAsk0A,463
85
+ cli_sdk/stats/venn_abers/_isotonic.py,sha256=ux3r0_CcjsuRj3hvCie8ZWgHxLd9R_CEueiQ9IhHqls,1473
86
+ cli_sdk/stats/venn_abers/cvap.py,sha256=f3fORbBC0ShnUyYNw747kcHpT4lLvR2fK9-BQ3jQ9TE,2356
87
+ cli_sdk/stats/venn_abers/ivap.py,sha256=Yua1H2OHiDUKw6SMHhEeDmx4sIsGzk40_wkvwZCGyNU,4650
88
+ cci_sdk-0.1.0.dist-info/METADATA,sha256=t5TKvtlBmZxuGHrrLurlIBXbyJrZL4Gs9Nh0dTcC6zs,16746
89
+ cci_sdk-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
90
+ cci_sdk-0.1.0.dist-info/entry_points.txt,sha256=3Habu-N3nF77zkm0IqAWenL3Fuvc300ucebniR74FgY,46
91
+ cci_sdk-0.1.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
92
+ cci_sdk-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.4
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ cli = cli_sdk.cli_tool:main