calibrated-agent-evaluation 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (416) hide show
  1. calibrated_agent_evaluation/__init__.py +16 -0
  2. calibrated_agent_evaluation/__main__.py +11 -0
  3. calibrated_agent_evaluation/_conversation.py +52 -0
  4. calibrated_agent_evaluation/_errors.py +47 -0
  5. calibrated_agent_evaluation/_filelock.py +55 -0
  6. calibrated_agent_evaluation/_finite.py +121 -0
  7. calibrated_agent_evaluation/_install.py +174 -0
  8. calibrated_agent_evaluation/_model.py +84 -0
  9. calibrated_agent_evaluation/_provenance.py +239 -0
  10. calibrated_agent_evaluation/_quickstart/__init__.py +132 -0
  11. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/compassion_fade.json +272 -0
  12. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/length.json +272 -0
  13. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/self_preference.json +380 -0
  14. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/sycophancy.json +284 -0
  15. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/compassion_fade.json +668 -0
  16. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/length.json +668 -0
  17. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/self_preference.json +938 -0
  18. calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/sycophancy.json +698 -0
  19. calibrated_agent_evaluation/_quickstart/data/certify/bias-thresholds.json +11 -0
  20. calibrated_agent_evaluation/_quickstart/data/certify/gold.jsonl.gz +0 -0
  21. calibrated_agent_evaluation/_quickstart/data/certify/judge-v1.json +8 -0
  22. calibrated_agent_evaluation/_quickstart/data/certify/judge-v2.json +8 -0
  23. calibrated_agent_evaluation/_quickstart/data/certify/judge.cassette.json.gz +0 -0
  24. calibrated_agent_evaluation/_quickstart/data/certify/manifest.json +43 -0
  25. calibrated_agent_evaluation/_quickstart/data/certify/traces.jsonl.gz +0 -0
  26. calibrated_agent_evaluation/_quickstart/data/labels/gold.json +132 -0
  27. calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4471.json +162 -0
  28. calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4473.json +150 -0
  29. calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4475.json +151 -0
  30. calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4477.json +158 -0
  31. calibrated_agent_evaluation/_quickstart/data/normalized/ORD-1001.json +71 -0
  32. calibrated_agent_evaluation/_quickstart/data/normalized/ORD-1002.json +71 -0
  33. calibrated_agent_evaluation/_recordstore.py +314 -0
  34. calibrated_agent_evaluation/agentcore/__init__.py +379 -0
  35. calibrated_agent_evaluation/agentcore/adot.py +330 -0
  36. calibrated_agent_evaluation/agentcore/arms.py +807 -0
  37. calibrated_agent_evaluation/agentcore/audit_report.py +347 -0
  38. calibrated_agent_evaluation/agentcore/audit_types.py +536 -0
  39. calibrated_agent_evaluation/agentcore/bias.py +320 -0
  40. calibrated_agent_evaluation/agentcore/calibrate.py +162 -0
  41. calibrated_agent_evaluation/agentcore/code_interpreter.py +260 -0
  42. calibrated_agent_evaluation/agentcore/cost.py +162 -0
  43. calibrated_agent_evaluation/agentcore/datasets.py +506 -0
  44. calibrated_agent_evaluation/agentcore/datasets_codec.py +219 -0
  45. calibrated_agent_evaluation/agentcore/detectors/__init__.py +11 -0
  46. calibrated_agent_evaluation/agentcore/detectors/_payload.py +134 -0
  47. calibrated_agent_evaluation/agentcore/detectors/detect.py +457 -0
  48. calibrated_agent_evaluation/agentcore/detectors/models.py +767 -0
  49. calibrated_agent_evaluation/agentcore/detectors/namer.py +376 -0
  50. calibrated_agent_evaluation/agentcore/detectors/session.py +347 -0
  51. calibrated_agent_evaluation/agentcore/detectors/types.py +216 -0
  52. calibrated_agent_evaluation/agentcore/detectors/vocabulary.py +373 -0
  53. calibrated_agent_evaluation/agentcore/doctor.py +513 -0
  54. calibrated_agent_evaluation/agentcore/endpoint_label.py +31 -0
  55. calibrated_agent_evaluation/agentcore/errors.py +568 -0
  56. calibrated_agent_evaluation/agentcore/evaluate.py +707 -0
  57. calibrated_agent_evaluation/agentcore/evaluate_payload.py +475 -0
  58. calibrated_agent_evaluation/agentcore/evaluator_create.py +403 -0
  59. calibrated_agent_evaluation/agentcore/guards.py +127 -0
  60. calibrated_agent_evaluation/agentcore/ingest.py +151 -0
  61. calibrated_agent_evaluation/agentcore/judge.py +583 -0
  62. calibrated_agent_evaluation/agentcore/metrics.py +254 -0
  63. calibrated_agent_evaluation/agentcore/online.py +807 -0
  64. calibrated_agent_evaluation/agentcore/online_record.py +331 -0
  65. calibrated_agent_evaluation/agentcore/online_version.py +15 -0
  66. calibrated_agent_evaluation/agentcore/optimization.py +525 -0
  67. calibrated_agent_evaluation/agentcore/policy_spans.py +466 -0
  68. calibrated_agent_evaluation/agentcore/pricing.py +349 -0
  69. calibrated_agent_evaluation/agentcore/readback.py +440 -0
  70. calibrated_agent_evaluation/agentcore/readers.py +795 -0
  71. calibrated_agent_evaluation/agentcore/replay.py +695 -0
  72. calibrated_agent_evaluation/agentcore/run_manifest.py +190 -0
  73. calibrated_agent_evaluation/agentcore/run_spans.py +106 -0
  74. calibrated_agent_evaluation/agentcore/runtime.py +1055 -0
  75. calibrated_agent_evaluation/agentcore/runtime_boundary.py +798 -0
  76. calibrated_agent_evaluation/agentcore/runtime_cassette.py +382 -0
  77. calibrated_agent_evaluation/agentcore/runtime_errors.py +49 -0
  78. calibrated_agent_evaluation/agentcore/runtime_executor.py +985 -0
  79. calibrated_agent_evaluation/agentcore/runtime_probe.py +306 -0
  80. calibrated_agent_evaluation/agentcore/runtime_response.py +307 -0
  81. calibrated_agent_evaluation/agentcore/scores.py +147 -0
  82. calibrated_agent_evaluation/agentcore/session.py +250 -0
  83. calibrated_agent_evaluation/agentcore/settle.py +96 -0
  84. calibrated_agent_evaluation/agentcore/simulated_user.py +139 -0
  85. calibrated_agent_evaluation/agentcore/sources.py +324 -0
  86. calibrated_agent_evaluation/agentcore/span_reader.py +399 -0
  87. calibrated_agent_evaluation/attribution/__init__.py +90 -0
  88. calibrated_agent_evaluation/attribution/agreement.py +107 -0
  89. calibrated_agent_evaluation/attribution/codebook.py +94 -0
  90. calibrated_agent_evaluation/attribution/definition.py +112 -0
  91. calibrated_agent_evaluation/attribution/errors.py +34 -0
  92. calibrated_agent_evaluation/attribution/labels.py +48 -0
  93. calibrated_agent_evaluation/attribution/rates.py +86 -0
  94. calibrated_agent_evaluation/attribution/relocation.py +139 -0
  95. calibrated_agent_evaluation/attribution/types.py +207 -0
  96. calibrated_agent_evaluation/attribution/walkback.py +190 -0
  97. calibrated_agent_evaluation/backends/__init__.py +79 -0
  98. calibrated_agent_evaluation/backends/_bounded.py +62 -0
  99. calibrated_agent_evaluation/backends/_export_tap.py +86 -0
  100. calibrated_agent_evaluation/backends/base.py +220 -0
  101. calibrated_agent_evaluation/backends/dashboard.py +334 -0
  102. calibrated_agent_evaluation/backends/factory.py +85 -0
  103. calibrated_agent_evaluation/backends/langfuse.py +1061 -0
  104. calibrated_agent_evaluation/backends/langfuse_foreign.py +210 -0
  105. calibrated_agent_evaluation/backends/local.py +322 -0
  106. calibrated_agent_evaluation/bias/__init__.py +80 -0
  107. calibrated_agent_evaluation/bias/battery.py +284 -0
  108. calibrated_agent_evaluation/bias/errors.py +131 -0
  109. calibrated_agent_evaluation/bias/gate.py +103 -0
  110. calibrated_agent_evaluation/bias/loader.py +120 -0
  111. calibrated_agent_evaluation/bias/mitigate.py +23 -0
  112. calibrated_agent_evaluation/bias/probe.py +170 -0
  113. calibrated_agent_evaluation/bias/record.py +123 -0
  114. calibrated_agent_evaluation/bias/store.py +62 -0
  115. calibrated_agent_evaluation/bias/thresholds.py +52 -0
  116. calibrated_agent_evaluation/bias/version.py +16 -0
  117. calibrated_agent_evaluation/calibrate/__init__.py +162 -0
  118. calibrated_agent_evaluation/calibrate/engine.py +570 -0
  119. calibrated_agent_evaluation/calibrate/errors.py +179 -0
  120. calibrated_agent_evaluation/calibrate/ledger.py +200 -0
  121. calibrated_agent_evaluation/calibrate/pairing.py +137 -0
  122. calibrated_agent_evaluation/calibrate/plan.py +411 -0
  123. calibrated_agent_evaluation/calibrate/record.py +282 -0
  124. calibrated_agent_evaluation/calibrate/store.py +67 -0
  125. calibrated_agent_evaluation/cli/__init__.py +31 -0
  126. calibrated_agent_evaluation/cli/agentcore_route.py +435 -0
  127. calibrated_agent_evaluation/cli/agentspec.py +43 -0
  128. calibrated_agent_evaluation/cli/args.py +1714 -0
  129. calibrated_agent_evaluation/cli/atoms.py +75 -0
  130. calibrated_agent_evaluation/cli/backends.py +168 -0
  131. calibrated_agent_evaluation/cli/config.py +146 -0
  132. calibrated_agent_evaluation/cli/consistency.py +461 -0
  133. calibrated_agent_evaluation/cli/conversation_route.py +107 -0
  134. calibrated_agent_evaluation/cli/dispatch.py +167 -0
  135. calibrated_agent_evaluation/cli/drift.py +206 -0
  136. calibrated_agent_evaluation/cli/errors.py +111 -0
  137. calibrated_agent_evaluation/cli/goldagreement.py +200 -0
  138. calibrated_agent_evaluation/cli/handlers.py +119 -0
  139. calibrated_agent_evaluation/cli/judgefile.py +233 -0
  140. calibrated_agent_evaluation/cli/judgekit.py +260 -0
  141. calibrated_agent_evaluation/cli/judging.py +213 -0
  142. calibrated_agent_evaluation/cli/pairing.py +324 -0
  143. calibrated_agent_evaluation/cli/parser.py +290 -0
  144. calibrated_agent_evaluation/cli/plain.py +558 -0
  145. calibrated_agent_evaluation/cli/present.py +104 -0
  146. calibrated_agent_evaluation/cli/proposals.py +58 -0
  147. calibrated_agent_evaluation/cli/records.py +655 -0
  148. calibrated_agent_evaluation/cli/scope.py +70 -0
  149. calibrated_agent_evaluation/cli/stages/__init__.py +9 -0
  150. calibrated_agent_evaluation/cli/stages/agreement.py +196 -0
  151. calibrated_agent_evaluation/cli/stages/audit.py +860 -0
  152. calibrated_agent_evaluation/cli/stages/bias.py +200 -0
  153. calibrated_agent_evaluation/cli/stages/calibrate.py +466 -0
  154. calibrated_agent_evaluation/cli/stages/calibrate_agentcore.py +311 -0
  155. calibrated_agent_evaluation/cli/stages/calibrate_check.py +269 -0
  156. calibrated_agent_evaluation/cli/stages/capture.py +52 -0
  157. calibrated_agent_evaluation/cli/stages/cluster.py +248 -0
  158. calibrated_agent_evaluation/cli/stages/codebook.py +110 -0
  159. calibrated_agent_evaluation/cli/stages/dataset.py +353 -0
  160. calibrated_agent_evaluation/cli/stages/dataset_run.py +481 -0
  161. calibrated_agent_evaluation/cli/stages/define.py +79 -0
  162. calibrated_agent_evaluation/cli/stages/doctor.py +259 -0
  163. calibrated_agent_evaluation/cli/stages/dumpload.py +186 -0
  164. calibrated_agent_evaluation/cli/stages/generate.py +679 -0
  165. calibrated_agent_evaluation/cli/stages/improve.py +1049 -0
  166. calibrated_agent_evaluation/cli/stages/ingest.py +434 -0
  167. calibrated_agent_evaluation/cli/stages/judge.py +336 -0
  168. calibrated_agent_evaluation/cli/stages/label.py +225 -0
  169. calibrated_agent_evaluation/cli/stages/label_pairwise.py +111 -0
  170. calibrated_agent_evaluation/cli/stages/online.py +359 -0
  171. calibrated_agent_evaluation/cli/stages/online_read.py +739 -0
  172. calibrated_agent_evaluation/cli/stages/plan.py +86 -0
  173. calibrated_agent_evaluation/cli/stages/prefer.py +232 -0
  174. calibrated_agent_evaluation/cli/stages/probe.py +656 -0
  175. calibrated_agent_evaluation/cli/stages/probe_policy.py +305 -0
  176. calibrated_agent_evaluation/cli/stages/quickstart.py +226 -0
  177. calibrated_agent_evaluation/cli/stages/ratify.py +131 -0
  178. calibrated_agent_evaluation/cli/stages/results.py +80 -0
  179. calibrated_agent_evaluation/cli/stages/run.py +851 -0
  180. calibrated_agent_evaluation/cli/stages/score.py +395 -0
  181. calibrated_agent_evaluation/cli/taxonomy.py +120 -0
  182. calibrated_agent_evaluation/cluster/__init__.py +49 -0
  183. calibrated_agent_evaluation/cluster/codebook.py +188 -0
  184. calibrated_agent_evaluation/cluster/coverage.py +64 -0
  185. calibrated_agent_evaluation/cluster/draft.py +73 -0
  186. calibrated_agent_evaluation/cluster/embed.py +40 -0
  187. calibrated_agent_evaluation/cluster/errors.py +16 -0
  188. calibrated_agent_evaluation/cluster/group.py +38 -0
  189. calibrated_agent_evaluation/cluster/ratify_map.py +302 -0
  190. calibrated_agent_evaluation/cluster/saturation.py +243 -0
  191. calibrated_agent_evaluation/cluster/types.py +87 -0
  192. calibrated_agent_evaluation/dumpload/__init__.py +76 -0
  193. calibrated_agent_evaluation/dumpload/io.py +576 -0
  194. calibrated_agent_evaluation/dumpload/registry.py +107 -0
  195. calibrated_agent_evaluation/dumpload/results.py +609 -0
  196. calibrated_agent_evaluation/enforce/__init__.py +107 -0
  197. calibrated_agent_evaluation/enforce/admission.py +130 -0
  198. calibrated_agent_evaluation/enforce/checklist.py +54 -0
  199. calibrated_agent_evaluation/enforce/context.py +28 -0
  200. calibrated_agent_evaluation/enforce/decision.py +67 -0
  201. calibrated_agent_evaluation/enforce/display.py +118 -0
  202. calibrated_agent_evaluation/enforce/errors.py +106 -0
  203. calibrated_agent_evaluation/enforce/gate.py +80 -0
  204. calibrated_agent_evaluation/enforce/judgedef.py +94 -0
  205. calibrated_agent_evaluation/enforce/lint.py +214 -0
  206. calibrated_agent_evaluation/enforce/store.py +101 -0
  207. calibrated_agent_evaluation/enforce/version.py +13 -0
  208. calibrated_agent_evaluation/execute/__init__.py +137 -0
  209. calibrated_agent_evaluation/execute/base.py +37 -0
  210. calibrated_agent_evaluation/execute/bedrock_embedder.py +123 -0
  211. calibrated_agent_evaluation/execute/bedrock_renderer.py +64 -0
  212. calibrated_agent_evaluation/execute/benchmark_tasks.py +50 -0
  213. calibrated_agent_evaluation/execute/cassette.py +459 -0
  214. calibrated_agent_evaluation/execute/cassette_provider.py +157 -0
  215. calibrated_agent_evaluation/execute/errors.py +43 -0
  216. calibrated_agent_evaluation/execute/inspect_executor.py +221 -0
  217. calibrated_agent_evaluation/execute/judge_model.py +459 -0
  218. calibrated_agent_evaluation/execute/probe_bridge.py +207 -0
  219. calibrated_agent_evaluation/execute/sandbox.py +274 -0
  220. calibrated_agent_evaluation/execute/strands_capture.py +323 -0
  221. calibrated_agent_evaluation/execute/strands_cassette_model.py +132 -0
  222. calibrated_agent_evaluation/execute/strands_executor.py +281 -0
  223. calibrated_agent_evaluation/execute/task_builders.py +142 -0
  224. calibrated_agent_evaluation/flagship/__init__.py +110 -0
  225. calibrated_agent_evaluation/flagship/errors.py +57 -0
  226. calibrated_agent_evaluation/flagship/oracle.py +112 -0
  227. calibrated_agent_evaluation/flagship/orchestrator.py +362 -0
  228. calibrated_agent_evaluation/flagship/reference.py +525 -0
  229. calibrated_agent_evaluation/flagship/report.py +199 -0
  230. calibrated_agent_evaluation/flagship/runner.py +183 -0
  231. calibrated_agent_evaluation/flagship/stages.py +292 -0
  232. calibrated_agent_evaluation/flagship/store.py +62 -0
  233. calibrated_agent_evaluation/improve/__init__.py +132 -0
  234. calibrated_agent_evaluation/improve/confirm.py +95 -0
  235. calibrated_agent_evaluation/improve/crew.py +344 -0
  236. calibrated_agent_evaluation/improve/errors.py +75 -0
  237. calibrated_agent_evaluation/improve/guard.py +244 -0
  238. calibrated_agent_evaluation/improve/paired.py +230 -0
  239. calibrated_agent_evaluation/improve/panel.py +196 -0
  240. calibrated_agent_evaluation/improve/prereg.py +263 -0
  241. calibrated_agent_evaluation/improve/report.py +311 -0
  242. calibrated_agent_evaluation/improve/store.py +161 -0
  243. calibrated_agent_evaluation/ingest/__init__.py +60 -0
  244. calibrated_agent_evaluation/ingest/core.py +1537 -0
  245. calibrated_agent_evaluation/ingest/detect.py +199 -0
  246. calibrated_agent_evaluation/ingest/errors.py +38 -0
  247. calibrated_agent_evaluation/ingest/sample.py +147 -0
  248. calibrated_agent_evaluation/judges/__init__.py +87 -0
  249. calibrated_agent_evaluation/judges/cache.py +140 -0
  250. calibrated_agent_evaluation/judges/errors.py +35 -0
  251. calibrated_agent_evaluation/judges/executable.py +92 -0
  252. calibrated_agent_evaluation/judges/levels.py +60 -0
  253. calibrated_agent_evaluation/judges/probe_runner.py +152 -0
  254. calibrated_agent_evaluation/judges/process.py +90 -0
  255. calibrated_agent_evaluation/judges/run_record.py +124 -0
  256. calibrated_agent_evaluation/judges/runtime.py +300 -0
  257. calibrated_agent_evaluation/judges/targets.py +163 -0
  258. calibrated_agent_evaluation/judges/types.py +56 -0
  259. calibrated_agent_evaluation/judges/verdict.py +83 -0
  260. calibrated_agent_evaluation/judges/verdict_schema.py +65 -0
  261. calibrated_agent_evaluation/judges/version.py +14 -0
  262. calibrated_agent_evaluation/normalize/__init__.py +185 -0
  263. calibrated_agent_evaluation/normalize/_depth.py +34 -0
  264. calibrated_agent_evaluation/normalize/agent_boundaries.py +65 -0
  265. calibrated_agent_evaluation/normalize/content.py +177 -0
  266. calibrated_agent_evaluation/normalize/content_placements.py +176 -0
  267. calibrated_agent_evaluation/normalize/conventions/__init__.py +41 -0
  268. calibrated_agent_evaluation/normalize/conventions/detect.py +121 -0
  269. calibrated_agent_evaluation/normalize/conventions/dispatch.py +31 -0
  270. calibrated_agent_evaluation/normalize/conventions/messages_adapter.py +269 -0
  271. calibrated_agent_evaluation/normalize/conventions/openinference_adapter.py +222 -0
  272. calibrated_agent_evaluation/normalize/conventions/openllmetry_adapter.py +260 -0
  273. calibrated_agent_evaluation/normalize/conventions/pairing.py +118 -0
  274. calibrated_agent_evaluation/normalize/conventions/registry.py +80 -0
  275. calibrated_agent_evaluation/normalize/datasets/__init__.py +154 -0
  276. calibrated_agent_evaluation/normalize/datasets/hf.py +78 -0
  277. calibrated_agent_evaluation/normalize/datasets/jsonl.py +44 -0
  278. calibrated_agent_evaluation/normalize/datasets/parquet.py +58 -0
  279. calibrated_agent_evaluation/normalize/errors.py +28 -0
  280. calibrated_agent_evaluation/normalize/guidance.py +70 -0
  281. calibrated_agent_evaluation/normalize/inspect_adapter.py +259 -0
  282. calibrated_agent_evaluation/normalize/otel_adapter.py +626 -0
  283. calibrated_agent_evaluation/normalize/otlp_json.py +363 -0
  284. calibrated_agent_evaluation/normalize/ownership.py +122 -0
  285. calibrated_agent_evaluation/normalize/sources.py +212 -0
  286. calibrated_agent_evaluation/normalize/strands_adapter.py +440 -0
  287. calibrated_agent_evaluation/normalize/tree.py +284 -0
  288. calibrated_agent_evaluation/probe/__init__.py +156 -0
  289. calibrated_agent_evaluation/probe/adapter.py +328 -0
  290. calibrated_agent_evaluation/probe/authorize.py +54 -0
  291. calibrated_agent_evaluation/probe/battery.py +180 -0
  292. calibrated_agent_evaluation/probe/benchmarks.py +214 -0
  293. calibrated_agent_evaluation/probe/capability.py +36 -0
  294. calibrated_agent_evaluation/probe/connection.py +139 -0
  295. calibrated_agent_evaluation/probe/errors.py +82 -0
  296. calibrated_agent_evaluation/probe/fingerprint.py +20 -0
  297. calibrated_agent_evaluation/probe/framing.py +29 -0
  298. calibrated_agent_evaluation/probe/harness.py +263 -0
  299. calibrated_agent_evaluation/probe/policy.py +561 -0
  300. calibrated_agent_evaluation/probe/policy_battery.py +296 -0
  301. calibrated_agent_evaluation/probe/rate.py +72 -0
  302. calibrated_agent_evaluation/probe/record.py +85 -0
  303. calibrated_agent_evaluation/probe/redact.py +32 -0
  304. calibrated_agent_evaluation/probe/robustness.py +174 -0
  305. calibrated_agent_evaluation/probe/store.py +61 -0
  306. calibrated_agent_evaluation/probe/version.py +8 -0
  307. calibrated_agent_evaluation/probe/ws_adapter.py +207 -0
  308. calibrated_agent_evaluation/py.typed +0 -0
  309. calibrated_agent_evaluation/review/__init__.py +150 -0
  310. calibrated_agent_evaluation/review/addresses.py +55 -0
  311. calibrated_agent_evaluation/review/agreement.py +255 -0
  312. calibrated_agent_evaluation/review/app.py +161 -0
  313. calibrated_agent_evaluation/review/context.py +254 -0
  314. calibrated_agent_evaluation/review/errors.py +45 -0
  315. calibrated_agent_evaluation/review/gate.py +36 -0
  316. calibrated_agent_evaluation/review/guards.py +152 -0
  317. calibrated_agent_evaluation/review/labeling.py +144 -0
  318. calibrated_agent_evaluation/review/overlap.py +241 -0
  319. calibrated_agent_evaluation/review/pairwise.py +339 -0
  320. calibrated_agent_evaluation/review/prefill.py +46 -0
  321. calibrated_agent_evaluation/review/proposals.py +43 -0
  322. calibrated_agent_evaluation/review/record.py +332 -0
  323. calibrated_agent_evaluation/review/relabel.py +36 -0
  324. calibrated_agent_evaluation/review/renderers/__init__.py +49 -0
  325. calibrated_agent_evaluation/review/renderers/base.py +97 -0
  326. calibrated_agent_evaluation/review/renderers/default.py +107 -0
  327. calibrated_agent_evaluation/review/renderers/trace_view.py +358 -0
  328. calibrated_agent_evaluation/review/routes.py +465 -0
  329. calibrated_agent_evaluation/review/session.py +269 -0
  330. calibrated_agent_evaluation/review/session_log.py +66 -0
  331. calibrated_agent_evaluation/review/static/hotkeys.js +385 -0
  332. calibrated_agent_evaluation/review/static/pairwise.js +28 -0
  333. calibrated_agent_evaluation/review/static/styles.css +130 -0
  334. calibrated_agent_evaluation/review/static/vendor/README.md +13 -0
  335. calibrated_agent_evaluation/review/static/vendor/htmx.min.js +1 -0
  336. calibrated_agent_evaluation/review/store.py +99 -0
  337. calibrated_agent_evaluation/run/__init__.py +143 -0
  338. calibrated_agent_evaluation/run/aggregate.py +75 -0
  339. calibrated_agent_evaluation/run/drift_report.py +333 -0
  340. calibrated_agent_evaluation/run/errors.py +54 -0
  341. calibrated_agent_evaluation/run/features.py +88 -0
  342. calibrated_agent_evaluation/run/flags.py +40 -0
  343. calibrated_agent_evaluation/run/gate.py +210 -0
  344. calibrated_agent_evaluation/run/judge_check.py +161 -0
  345. calibrated_agent_evaluation/run/multi.py +41 -0
  346. calibrated_agent_evaluation/run/report.py +457 -0
  347. calibrated_agent_evaluation/run/staleness.py +81 -0
  348. calibrated_agent_evaluation/run/store.py +61 -0
  349. calibrated_agent_evaluation/run/trend.py +161 -0
  350. calibrated_agent_evaluation/sampling/__init__.py +49 -0
  351. calibrated_agent_evaluation/sampling/discovery.py +161 -0
  352. calibrated_agent_evaluation/sampling/errors.py +26 -0
  353. calibrated_agent_evaluation/sampling/production.py +192 -0
  354. calibrated_agent_evaluation/sampling/stats.py +44 -0
  355. calibrated_agent_evaluation/sampling/types.py +48 -0
  356. calibrated_agent_evaluation/sampling/validate.py +53 -0
  357. calibrated_agent_evaluation/scenarios/__init__.py +56 -0
  358. calibrated_agent_evaluation/scenarios/build.py +224 -0
  359. calibrated_agent_evaluation/scenarios/errors.py +14 -0
  360. calibrated_agent_evaluation/scenarios/labels.py +49 -0
  361. calibrated_agent_evaluation/scenarios/slicefile.py +101 -0
  362. calibrated_agent_evaluation/scenarios/types.py +212 -0
  363. calibrated_agent_evaluation/schema/__init__.py +70 -0
  364. calibrated_agent_evaluation/schema/errors.py +58 -0
  365. calibrated_agent_evaluation/schema/serde.py +182 -0
  366. calibrated_agent_evaluation/schema/types.py +174 -0
  367. calibrated_agent_evaluation/schema/validate.py +213 -0
  368. calibrated_agent_evaluation/scorers/__init__.py +86 -0
  369. calibrated_agent_evaluation/scorers/base.py +47 -0
  370. calibrated_agent_evaluation/scorers/batch.py +51 -0
  371. calibrated_agent_evaluation/scorers/errors.py +25 -0
  372. calibrated_agent_evaluation/scorers/execution.py +475 -0
  373. calibrated_agent_evaluation/scorers/heuristic.py +312 -0
  374. calibrated_agent_evaluation/scorers/lane.py +28 -0
  375. calibrated_agent_evaluation/scorers/passk.py +129 -0
  376. calibrated_agent_evaluation/stats/__init__.py +55 -0
  377. calibrated_agent_evaluation/stats/agreement.py +311 -0
  378. calibrated_agent_evaluation/stats/drift.py +353 -0
  379. calibrated_agent_evaluation/stats/kappa.py +106 -0
  380. calibrated_agent_evaluation/stats/mcnemar.py +178 -0
  381. calibrated_agent_evaluation/stats/multiplicity.py +60 -0
  382. calibrated_agent_evaluation/stats/proportions.py +137 -0
  383. calibrated_agent_evaluation/stats/rogan_gladen.py +342 -0
  384. calibrated_agent_evaluation/stats/studies/__init__.py +19 -0
  385. calibrated_agent_evaluation/stats/studies/baseline_arm.py +143 -0
  386. calibrated_agent_evaluation/stats/studies/coverage.py +325 -0
  387. calibrated_agent_evaluation/stats/studies/planner_floor.py +160 -0
  388. calibrated_agent_evaluation/stats/studies/rate_sample_minimum.py +147 -0
  389. calibrated_agent_evaluation/subjective/__init__.py +70 -0
  390. calibrated_agent_evaluation/subjective/atomfile.py +86 -0
  391. calibrated_agent_evaluation/subjective/bradley_terry.py +192 -0
  392. calibrated_agent_evaluation/subjective/classify.py +68 -0
  393. calibrated_agent_evaluation/subjective/elo.py +96 -0
  394. calibrated_agent_evaluation/subjective/errors.py +20 -0
  395. calibrated_agent_evaluation/subjective/gate.py +29 -0
  396. calibrated_agent_evaluation/subjective/pairfile.py +91 -0
  397. calibrated_agent_evaluation/subjective/preference.py +49 -0
  398. calibrated_agent_evaluation/subjective/store.py +142 -0
  399. calibrated_agent_evaluation/subjective/types.py +99 -0
  400. calibrated_agent_evaluation/synth/__init__.py +50 -0
  401. calibrated_agent_evaluation/synth/borrow/__init__.py +13 -0
  402. calibrated_agent_evaluation/synth/borrow/simulator.py +247 -0
  403. calibrated_agent_evaluation/synth/conversation.py +460 -0
  404. calibrated_agent_evaluation/synth/conversation_cassette.py +248 -0
  405. calibrated_agent_evaluation/synth/errors.py +83 -0
  406. calibrated_agent_evaluation/synth/generate.py +352 -0
  407. calibrated_agent_evaluation/synth/guards.py +115 -0
  408. calibrated_agent_evaluation/synth/risk.py +75 -0
  409. calibrated_agent_evaluation/synth/types.py +78 -0
  410. calibrated_agent_evaluation-0.1.0.dist-info/METADATA +237 -0
  411. calibrated_agent_evaluation-0.1.0.dist-info/RECORD +416 -0
  412. calibrated_agent_evaluation-0.1.0.dist-info/WHEEL +4 -0
  413. calibrated_agent_evaluation-0.1.0.dist-info/entry_points.txt +2 -0
  414. calibrated_agent_evaluation-0.1.0.dist-info/licenses/LICENSE +175 -0
  415. calibrated_agent_evaluation-0.1.0.dist-info/licenses/NOTICE +2 -0
  416. calibrated_agent_evaluation-0.1.0.dist-info/licenses/THIRD-PARTY-LICENSES +446 -0
@@ -0,0 +1,16 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Calibrated Agent Evaluation: error analysis and judge calibration for AI agents.
5
+
6
+ The distribution and the command-line entry point are both ``calibrated-agent-evaluation``.
7
+ """
8
+
9
+ from importlib.metadata import PackageNotFoundError, version
10
+
11
+ try:
12
+ __version__ = version("calibrated-agent-evaluation")
13
+ except PackageNotFoundError: # pragma: no cover - package not installed
14
+ __version__ = "0.0.0+unknown"
15
+
16
+ __all__ = ["__version__"]
@@ -0,0 +1,11 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """``python -m calibrated_agent_evaluation``: the ``calibrated-agent-evaluation`` command."""
5
+
6
+ from __future__ import annotations
7
+
8
+ from calibrated_agent_evaluation.cli import main
9
+
10
+ if __name__ == "__main__":
11
+ raise SystemExit(main())
@@ -0,0 +1,52 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """The value types both sides of a simulated conversation share, kept outside the ``synth`` plane.
5
+
6
+ The AgentCore adapters need these types at run time (the Runtime executor checks a task's
7
+ :class:`ConversationTurn`, the AgentCore simulated user returns a :class:`UserTurn`), and the
8
+ ``[agentcore]`` extra does not install the sampling stack that importing anything under
9
+ ``calibrated_agent_evaluation.synth`` loads. So the types live here, and
10
+ :mod:`calibrated_agent_evaluation.synth.conversation` re-exports them under its own names.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import dataclass
16
+ from typing import Literal, Protocol
17
+
18
+ from calibrated_agent_evaluation._model import CalibratedAgentEvaluationModel
19
+
20
+ __all__ = ["ConversationTurn", "SimulatedUser", "StopReason", "UserTurn"]
21
+
22
+ StopReason = Literal["goal_completed", "actor_stopped", "max_turns"]
23
+
24
+
25
+ class UserTurn(CalibratedAgentEvaluationModel):
26
+ """The simulated user's next move: a message, or a stop with its reason.
27
+
28
+ A stop may carry a last message (a customer who says "Thanks, that's all" and leaves); the
29
+ conversation keeps it, and no agent turn follows it. A turn with ``stop=False`` but no message
30
+ is read as a stop (``actor_stopped``): there is no customer message to send, and inventing one
31
+ would put a turn in the transcript nobody took.
32
+ """
33
+
34
+ message: str | None
35
+ stop: bool
36
+ stop_reason: StopReason | None = None
37
+
38
+
39
+ class SimulatedUser(Protocol):
40
+ """The port a simulated-user adapter implements."""
41
+
42
+ def act(self, agent_message: str) -> UserTurn:
43
+ """Answer the agent's latest reply with the next customer move."""
44
+ ...
45
+
46
+
47
+ @dataclass(frozen=True)
48
+ class ConversationTurn:
49
+ """Which conversation and which agent turn (1-based) a task belongs to."""
50
+
51
+ conversation_id: str
52
+ turn: int
@@ -0,0 +1,47 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """The root of the library's typed error hierarchy.
5
+
6
+ It lives outside :mod:`calibrated_agent_evaluation.schema` so that the model base in
7
+ :mod:`calibrated_agent_evaluation._model`, which the schema types themselves build on, can name it
8
+ without importing the schema package.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ from typing import TYPE_CHECKING
14
+
15
+ if TYPE_CHECKING:
16
+ from calibrated_agent_evaluation._install import InstallSource
17
+
18
+ __all__ = ["CalibratedAgentEvaluationDataError", "MissingExtraError"]
19
+
20
+
21
+ class CalibratedAgentEvaluationDataError(Exception):
22
+ """Base class for every data-plane error."""
23
+
24
+
25
+ class MissingExtraError(CalibratedAgentEvaluationDataError):
26
+ """A capability was asked for whose optional extra is not installed.
27
+
28
+ ``hint`` is the one command that installs ``extras`` into this installation, matched to how
29
+ Calibrated Agent Evaluation was installed (checkout, editable, uv tool, wheel or index).
30
+ """
31
+
32
+ def __init__(
33
+ self, extras: tuple[str, ...], purpose: str, *, source: InstallSource | None = None
34
+ ) -> None:
35
+ # Imported here: _install builds on pydantic, and this module must stay importable first.
36
+ from calibrated_agent_evaluation._install import _running_source, hint_for
37
+
38
+ self.extras = extras
39
+ self.hint = hint_for(source or _running_source(), extras)
40
+ names = ", ".join(f"[{e}]" for e in extras)
41
+ super().__init__(f"{purpose} needs the {names} extra; install it with: {self.hint}")
42
+
43
+
44
+ # Public home is calibrated_agent_evaluation.schema.errors, which re-exports them; tracebacks and
45
+ # the reference say so.
46
+ CalibratedAgentEvaluationDataError.__module__ = "calibrated_agent_evaluation.schema.errors"
47
+ MissingExtraError.__module__ = "calibrated_agent_evaluation.schema.errors"
@@ -0,0 +1,55 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """An advisory, cross-process exclusive lock on a sidecar file.
5
+
6
+ ``fcntl.flock`` on POSIX, ``msvcrt.locking`` on Windows. Advisory: it serializes cooperating
7
+ writers (two ``calibrated-agent-evaluation`` processes recording one cassette), not arbitrary
8
+ programs. Not reentrant.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import os
14
+ from collections.abc import Iterator
15
+ from contextlib import contextmanager
16
+ from pathlib import Path
17
+
18
+ __all__ = ["exclusive_lock"]
19
+
20
+
21
+ @contextmanager
22
+ def exclusive_lock(lock_path: Path) -> Iterator[None]:
23
+ """Hold an exclusive lock on ``lock_path`` (created if absent) for the ``with`` block.
24
+
25
+ Raises:
26
+ OSError: The lock file cannot be created or locked.
27
+ """
28
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
29
+ with lock_path.open("a+b") as fh:
30
+ _lock(fh.fileno())
31
+ try:
32
+ yield
33
+ finally:
34
+ _unlock(fh.fileno())
35
+
36
+
37
+ if os.name == "nt": # pragma: no cover - exercised on Windows only
38
+ import msvcrt
39
+
40
+ def _lock(fd: int) -> None:
41
+ os.lseek(fd, 0, os.SEEK_SET)
42
+ msvcrt.locking(fd, msvcrt.LK_LOCK, 1)
43
+
44
+ def _unlock(fd: int) -> None:
45
+ os.lseek(fd, 0, os.SEEK_SET)
46
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
47
+
48
+ else:
49
+ import fcntl
50
+
51
+ def _lock(fd: int) -> None:
52
+ fcntl.flock(fd, fcntl.LOCK_EX)
53
+
54
+ def _unlock(fd: int) -> None:
55
+ fcntl.flock(fd, fcntl.LOCK_UN)
@@ -0,0 +1,121 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """Refuse a value that standard JSON cannot represent, by one rule on every supported Python.
5
+
6
+ Two things make a value one that JSON cannot represent here:
7
+
8
+ - **A NaN or infinite number.** Python's ``json`` writes and reads the ``NaN``, ``Infinity`` and
9
+ ``-Infinity`` tokens unless told not to, but they are not JSON. A literal too large for a float,
10
+ such as ``1e999``, decodes to an infinity with no token, so a decoded value is checked too.
11
+ - **Nesting deeper than** :data:`MAX_JSON_DEPTH` **levels.** How deep Python's encoder and decoder
12
+ go before ``RecursionError`` differs between versions, so a fixed limit, checked before any
13
+ encoder runs, gives one answer everywhere.
14
+
15
+ :func:`require_finite` checks a value for both, and :func:`refuse_json_constant` is the
16
+ ``parse_constant`` hook that refuses the three tokens during a decode. Both raise ``ValueError``
17
+ subclasses, so a caller's existing "not JSON" handling catches them. Their messages name where the
18
+ value came from and the rule it broke, never a key or a value of it.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import math
24
+ from collections.abc import Mapping
25
+ from typing import Final, NoReturn
26
+
27
+ __all__ = [
28
+ "MAX_JSON_DEPTH",
29
+ "NestedTooDeepError",
30
+ "NonFiniteNumberError",
31
+ "refuse_json_constant",
32
+ "require_finite",
33
+ ]
34
+
35
+ MAX_JSON_DEPTH: Final = 256
36
+ """The deepest nesting a value may have.
37
+
38
+ A value's depth is the number of containers (mappings, lists, tuples, sets and frozensets) on its
39
+ deepest path, the outermost included: a scalar has depth 0, ``[]`` and ``{"a": 1}`` have depth 1,
40
+ and ``[[1]]`` has depth 2. A mapping's keys count as items of the mapping. Each caller counts from
41
+ the value it checks.
42
+ """
43
+
44
+
45
+ class NonFiniteNumberError(ValueError):
46
+ """A value holds a NaN or infinite number, which standard JSON has no form for."""
47
+
48
+
49
+ class NestedTooDeepError(ValueError):
50
+ """A value nests more than :data:`MAX_JSON_DEPTH` levels deep."""
51
+
52
+
53
+ def require_finite(value: object, *, where: str) -> None:
54
+ """Refuse ``value`` if it holds a NaN or infinite number, or nests too deep.
55
+
56
+ Walks mapping keys and values and the items of lists, tuples, sets and frozensets, stopping at
57
+ the first problem. It keeps its own stack and never looks inside a container past the limit,
58
+ so it cannot raise ``RecursionError``. Any other type is left to the caller's encoder.
59
+
60
+ Args:
61
+ value: The value to check, its depth counted from here.
62
+ where: What the value is, such as ``"request body"``; the message names it.
63
+
64
+ Raises:
65
+ NonFiniteNumberError: A ``float`` in the value is NaN or infinite.
66
+ NestedTooDeepError: A container in the value sits deeper than :data:`MAX_JSON_DEPTH`
67
+ levels.
68
+ """
69
+ # Each item with the level of the container holding it: 0 for the value itself.
70
+ pending: list[tuple[object, int]] = [(value, 0)]
71
+ while pending:
72
+ item, holder_level = pending.pop()
73
+ if isinstance(item, float):
74
+ if not math.isfinite(item):
75
+ raise NonFiniteNumberError(
76
+ f"{where} holds a number JSON cannot represent ({_kind(item)})"
77
+ )
78
+ continue
79
+ children = _children(item)
80
+ if children is None:
81
+ continue
82
+ level = holder_level + 1
83
+ if level > MAX_JSON_DEPTH:
84
+ raise NestedTooDeepError(f"{where} nests more than {MAX_JSON_DEPTH} levels deep")
85
+ # Reversed, so items are met in the order the value holds them.
86
+ pending.extend((child, level) for child in reversed(children))
87
+
88
+
89
+ def refuse_json_constant(token: str) -> NoReturn:
90
+ """The ``parse_constant`` hook of a standard-JSON decode: refuse every constant it is given.
91
+
92
+ ``json.loads`` calls it only for the ``NaN``, ``Infinity`` and ``-Infinity`` tokens.
93
+
94
+ Args:
95
+ token: The token the decoder met.
96
+
97
+ Raises:
98
+ NonFiniteNumberError: Always, naming the token.
99
+ """
100
+ # Python spells the three tokens the way float() reads them.
101
+ raise NonFiniteNumberError(
102
+ f"the JSON text holds the token {token}, a number JSON cannot represent "
103
+ f"({_kind(float(token))})"
104
+ )
105
+
106
+
107
+ def _kind(number: float) -> str:
108
+ return "NaN" if math.isnan(number) else "infinite"
109
+
110
+
111
+ def _children(item: object) -> list[object] | None:
112
+ # A container's items, keys included; None for anything that is not a container.
113
+ if isinstance(item, Mapping):
114
+ children: list[object] = []
115
+ for key, child in item.items():
116
+ children.append(key)
117
+ children.append(child)
118
+ return children
119
+ if isinstance(item, list | tuple | set | frozenset):
120
+ return list(item)
121
+ return None
@@ -0,0 +1,174 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """The command that adds an optional extra to the running installation.
5
+
6
+ Every "this needs an extra" message renders its install command here, so the command matches how
7
+ the package was installed: the checkout's ``uv sync``, a ``uv tool`` reinstall, or an install into
8
+ the running interpreter's environment, with pip when pip installed it and uv otherwise. Arguments
9
+ are quoted for the platform's shell.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import os
16
+ import shlex
17
+ import subprocess
18
+ import sys
19
+ import tomllib
20
+ from functools import cache
21
+ from importlib import metadata
22
+ from pathlib import Path
23
+ from typing import Literal, Protocol
24
+ from urllib.parse import urlparse
25
+ from urllib.request import url2pathname
26
+
27
+ from pydantic import BaseModel, ConfigDict
28
+
29
+ __all__ = ["InstallSource", "detect_install_source", "hint_for", "install_hint"]
30
+
31
+ _DISTRIBUTION = "calibrated-agent-evaluation"
32
+
33
+
34
+ class _Distribution(Protocol):
35
+ def read_text(self, filename: str) -> str | None: ...
36
+
37
+
38
+ class InstallSource(BaseModel):
39
+ """How the running Calibrated Agent Evaluation was installed."""
40
+
41
+ model_config = ConfigDict(frozen=True)
42
+
43
+ kind: Literal["project", "editable", "tool", "local", "index"]
44
+ path: str | None = None
45
+ """The wheel or directory a ``tool`` or ``local`` install came from."""
46
+ tool_extras: tuple[str, ...] = ()
47
+ """Extras already in a ``uv tool`` environment, which a reinstall must repeat to keep."""
48
+ installer: str | None = None
49
+ """The tool the distribution's ``INSTALLER`` file names (``pip``, ``uv``), when it has one."""
50
+ python: str | None = None
51
+ """The interpreter of the environment, which an install command has to target."""
52
+
53
+
54
+ def _direct_url(dist: _Distribution) -> dict[str, object]:
55
+ raw = dist.read_text("direct_url.json")
56
+ if raw is None:
57
+ return {}
58
+ try:
59
+ parsed = json.loads(raw)
60
+ except json.JSONDecodeError:
61
+ return {}
62
+ return parsed if isinstance(parsed, dict) else {}
63
+
64
+
65
+ def _tool_receipt(prefix: Path) -> tuple[tuple[str, ...], str | None] | None:
66
+ """The extras and source path recorded in a ``uv tool`` environment, or None outside one."""
67
+ receipt = prefix / "uv-receipt.toml"
68
+ if not receipt.is_file():
69
+ return None
70
+ try:
71
+ doc = tomllib.loads(receipt.read_text(encoding="utf-8"))
72
+ except (OSError, tomllib.TOMLDecodeError):
73
+ return (), None
74
+ for req in doc.get("tool", {}).get("requirements", []):
75
+ if isinstance(req, dict) and req.get("name") == _DISTRIBUTION:
76
+ return tuple(req.get("extras", ())), req.get("path")
77
+ return (), None
78
+
79
+
80
+ def detect_install_source(
81
+ dist: _Distribution | None, prefix: Path, python: str | None = None
82
+ ) -> InstallSource:
83
+ """Classify an installation from its distribution metadata and environment prefix.
84
+
85
+ Args:
86
+ dist: The installed ``calibrated-agent-evaluation`` distribution, or None when it was
87
+ imported from a source tree that was never installed.
88
+ prefix: The environment's ``sys.prefix``.
89
+ python: The environment's interpreter, which the install command targets.
90
+ """
91
+ if dist is None:
92
+ return InstallSource(kind="editable", python=python)
93
+ installer = (dist.read_text("INSTALLER") or "").strip().lower() or None
94
+ source = _classify(dist, prefix)
95
+ return source.model_copy(update={"installer": installer, "python": python})
96
+
97
+
98
+ def _classify(dist: _Distribution, prefix: Path) -> InstallSource:
99
+ direct = _direct_url(dist)
100
+ url = str(direct.get("url", ""))
101
+ # A file URL is percent-encoded: a checkout path with a space arrives as `%20`.
102
+ local_path = url2pathname(urlparse(url).path) if url.startswith("file://") else None
103
+ dir_info = direct.get("dir_info")
104
+ if isinstance(dir_info, dict) and dir_info.get("editable"):
105
+ # `uv sync` installs the checkout into <checkout>/.venv; any other editable install was
106
+ # made into some other environment by hand.
107
+ if local_path and prefix.resolve() == (Path(local_path) / ".venv").resolve():
108
+ return InstallSource(kind="project")
109
+ return InstallSource(kind="editable")
110
+ receipt = _tool_receipt(prefix)
111
+ if receipt is not None:
112
+ extras, receipt_path = receipt
113
+ return InstallSource(kind="tool", path=receipt_path or local_path, tool_extras=extras)
114
+ if local_path:
115
+ return InstallSource(kind="local", path=local_path)
116
+ return InstallSource(kind="index")
117
+
118
+
119
+ def _quote(arg: str) -> str:
120
+ """``arg`` as one word of the platform's shell."""
121
+ return subprocess.list2cmdline([arg]) if os.name == "nt" else shlex.quote(arg)
122
+
123
+
124
+ def hint_for(source: InstallSource, extras: tuple[str, ...]) -> str:
125
+ """The command that adds ``extras`` to an installation of the given kind.
126
+
127
+ ``uv sync`` removes packages it was not asked for, so the project form passes ``--inexact``.
128
+ ``uv tool install`` replaces the tool's extras, so the tool form repeats the ones it has.
129
+ A path with extras has to be written as a direct reference (``name[x] @ path``); uv rejects
130
+ ``path[x]``. Any other install is into the running interpreter's environment: a bare
131
+ ``uv pip install`` targets whichever environment uv finds first, so the uv form names the
132
+ interpreter with ``--python``, and an install pip made gets ``<python> -m pip install``.
133
+ """
134
+ names = ",".join(extras)
135
+ if source.kind == "project":
136
+ return "uv sync --inexact " + " ".join(f"--extra {e}" for e in extras)
137
+ if source.kind == "tool":
138
+ merged = ",".join(dict.fromkeys((*source.tool_extras, *extras)))
139
+ spec = f"{_DISTRIBUTION}[{merged}]"
140
+ return "uv tool install " + _quote(f"{spec} @ {source.path}" if source.path else spec)
141
+ if source.kind == "editable":
142
+ target = ["-e", _quote(f".[{names}]")]
143
+ elif source.kind == "local" and source.path:
144
+ target = [_quote(f"{_DISTRIBUTION}[{names}] @ {source.path}")]
145
+ else:
146
+ target = [_quote(f"{_DISTRIBUTION}[{names}]")]
147
+ if source.installer == "pip":
148
+ return " ".join([_quote(source.python or "python"), "-m", "pip", "install", *target])
149
+ python = ["--python", _quote(source.python)] if source.python else []
150
+ return " ".join(["uv", "pip", "install", *python, *target])
151
+
152
+
153
+ @cache
154
+ def _running_source() -> InstallSource:
155
+ try:
156
+ dist: _Distribution | None = metadata.distribution(_DISTRIBUTION)
157
+ except metadata.PackageNotFoundError:
158
+ dist = None
159
+ return detect_install_source(dist, Path(sys.prefix), sys.executable)
160
+
161
+
162
+ def install_hint(*extras: str) -> str:
163
+ """The shell command that installs ``extras`` into this installation.
164
+
165
+ Args:
166
+ extras: The optional extras to add, by their ``pyproject`` names.
167
+
168
+ Returns:
169
+ One command: ``uv sync --inexact --extra X`` in the checkout's own environment,
170
+ ``uv tool install`` for a uv tool, and otherwise ``uv pip install --python PYTHON`` (or
171
+ ``PYTHON -m pip install`` when pip installed it) with ``-e '.[X]'`` for another editable
172
+ install or the distribution, as a direct reference to a local wheel or from an index.
173
+ """
174
+ return hint_for(_running_source(), extras)
@@ -0,0 +1,84 @@
1
+ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ """The base for the library's pydantic models, and a validated copy-with-changes.
5
+
6
+ Every record and wire type subclasses :class:`CalibratedAgentEvaluationModel`, which turns a
7
+ pydantic ``ValidationError`` raised while building an instance into the model family's own typed
8
+ error (``invalid_error``), so a caller catches ``CalibrationError`` or ``RunError`` and never a raw
9
+ pydantic exception.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import dataclasses
15
+ from typing import Any, ClassVar, Self, TypeVar
16
+
17
+ from pydantic import (
18
+ BaseModel,
19
+ ConfigDict,
20
+ ModelWrapValidatorHandler,
21
+ ValidationError,
22
+ model_validator,
23
+ )
24
+
25
+ from calibrated_agent_evaluation._errors import CalibratedAgentEvaluationDataError
26
+
27
+ __all__ = ["CalibratedAgentEvaluationModel", "plain_error", "replace"]
28
+
29
+ T = TypeVar("T")
30
+
31
+
32
+ def plain_error(exc: ValidationError) -> str:
33
+ """Summarize a validation failure as ``field: problem`` pairs, without pydantic's URLs."""
34
+ parts = []
35
+ for err in exc.errors(include_url=False):
36
+ where = ".".join(str(p) for p in err["loc"]) or "value"
37
+ parts.append(f"{where}: {err['msg']}")
38
+ return "; ".join(parts)
39
+
40
+
41
+ class CalibratedAgentEvaluationModel(BaseModel):
42
+ """A pydantic model whose invalid input raises the family's typed error.
43
+
44
+ Subclasses set ``invalid_error`` to their package's base error and declare their own
45
+ ``model_config`` (frozen, ``extra="forbid"``).
46
+ """
47
+
48
+ # Building every model's schema at import costs each CLI command ~70 ms before it does
49
+ # anything; deferring builds a schema the first time its model is used.
50
+ model_config = ConfigDict(defer_build=True)
51
+
52
+ invalid_error: ClassVar[type[Exception]] = CalibratedAgentEvaluationDataError
53
+
54
+ @model_validator(mode="wrap")
55
+ @classmethod
56
+ def _raise_typed(cls, data: Any, handler: ModelWrapValidatorHandler[Self]) -> Self:
57
+ try:
58
+ return handler(data)
59
+ except ValidationError as exc:
60
+ raise cls.invalid_error(f"invalid {cls.__name__}: {plain_error(exc)}") from exc
61
+
62
+
63
+ def replace(obj: T, /, **changes: Any) -> T:
64
+ """Return a copy of ``obj`` with ``changes`` applied, validated like a fresh construction.
65
+
66
+ Unlike ``BaseModel.model_copy(update=...)``, the copy goes through the model's validators, so
67
+ a change that breaks an invariant raises exactly as constructing it would.
68
+
69
+ Args:
70
+ obj: A pydantic model or a dataclass instance.
71
+ **changes: Field values to replace.
72
+
73
+ Returns:
74
+ The new instance.
75
+
76
+ Raises:
77
+ TypeError: ``obj`` is neither a pydantic model nor a dataclass instance.
78
+ """
79
+ if isinstance(obj, BaseModel):
80
+ fields = {name: getattr(obj, name) for name in type(obj).model_fields}
81
+ return type(obj)(**{**fields, **changes})
82
+ if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
83
+ return dataclasses.replace(obj, **changes)
84
+ raise TypeError(f"replace() needs a pydantic model or a dataclass, got {type(obj).__name__}")