calibrated-agent-evaluation 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- calibrated_agent_evaluation/__init__.py +16 -0
- calibrated_agent_evaluation/__main__.py +11 -0
- calibrated_agent_evaluation/_conversation.py +52 -0
- calibrated_agent_evaluation/_errors.py +47 -0
- calibrated_agent_evaluation/_filelock.py +55 -0
- calibrated_agent_evaluation/_finite.py +121 -0
- calibrated_agent_evaluation/_install.py +174 -0
- calibrated_agent_evaluation/_model.py +84 -0
- calibrated_agent_evaluation/_provenance.py +239 -0
- calibrated_agent_evaluation/_quickstart/__init__.py +132 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/compassion_fade.json +272 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/length.json +272 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/self_preference.json +380 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-1/sycophancy.json +284 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/compassion_fade.json +668 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/length.json +668 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/self_preference.json +938 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-probes-2/sycophancy.json +698 -0
- calibrated_agent_evaluation/_quickstart/data/certify/bias-thresholds.json +11 -0
- calibrated_agent_evaluation/_quickstart/data/certify/gold.jsonl.gz +0 -0
- calibrated_agent_evaluation/_quickstart/data/certify/judge-v1.json +8 -0
- calibrated_agent_evaluation/_quickstart/data/certify/judge-v2.json +8 -0
- calibrated_agent_evaluation/_quickstart/data/certify/judge.cassette.json.gz +0 -0
- calibrated_agent_evaluation/_quickstart/data/certify/manifest.json +43 -0
- calibrated_agent_evaluation/_quickstart/data/certify/traces.jsonl.gz +0 -0
- calibrated_agent_evaluation/_quickstart/data/labels/gold.json +132 -0
- calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4471.json +162 -0
- calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4473.json +150 -0
- calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4475.json +151 -0
- calibrated_agent_evaluation/_quickstart/data/normalized/CLM-4477.json +158 -0
- calibrated_agent_evaluation/_quickstart/data/normalized/ORD-1001.json +71 -0
- calibrated_agent_evaluation/_quickstart/data/normalized/ORD-1002.json +71 -0
- calibrated_agent_evaluation/_recordstore.py +314 -0
- calibrated_agent_evaluation/agentcore/__init__.py +379 -0
- calibrated_agent_evaluation/agentcore/adot.py +330 -0
- calibrated_agent_evaluation/agentcore/arms.py +807 -0
- calibrated_agent_evaluation/agentcore/audit_report.py +347 -0
- calibrated_agent_evaluation/agentcore/audit_types.py +536 -0
- calibrated_agent_evaluation/agentcore/bias.py +320 -0
- calibrated_agent_evaluation/agentcore/calibrate.py +162 -0
- calibrated_agent_evaluation/agentcore/code_interpreter.py +260 -0
- calibrated_agent_evaluation/agentcore/cost.py +162 -0
- calibrated_agent_evaluation/agentcore/datasets.py +506 -0
- calibrated_agent_evaluation/agentcore/datasets_codec.py +219 -0
- calibrated_agent_evaluation/agentcore/detectors/__init__.py +11 -0
- calibrated_agent_evaluation/agentcore/detectors/_payload.py +134 -0
- calibrated_agent_evaluation/agentcore/detectors/detect.py +457 -0
- calibrated_agent_evaluation/agentcore/detectors/models.py +767 -0
- calibrated_agent_evaluation/agentcore/detectors/namer.py +376 -0
- calibrated_agent_evaluation/agentcore/detectors/session.py +347 -0
- calibrated_agent_evaluation/agentcore/detectors/types.py +216 -0
- calibrated_agent_evaluation/agentcore/detectors/vocabulary.py +373 -0
- calibrated_agent_evaluation/agentcore/doctor.py +513 -0
- calibrated_agent_evaluation/agentcore/endpoint_label.py +31 -0
- calibrated_agent_evaluation/agentcore/errors.py +568 -0
- calibrated_agent_evaluation/agentcore/evaluate.py +707 -0
- calibrated_agent_evaluation/agentcore/evaluate_payload.py +475 -0
- calibrated_agent_evaluation/agentcore/evaluator_create.py +403 -0
- calibrated_agent_evaluation/agentcore/guards.py +127 -0
- calibrated_agent_evaluation/agentcore/ingest.py +151 -0
- calibrated_agent_evaluation/agentcore/judge.py +583 -0
- calibrated_agent_evaluation/agentcore/metrics.py +254 -0
- calibrated_agent_evaluation/agentcore/online.py +807 -0
- calibrated_agent_evaluation/agentcore/online_record.py +331 -0
- calibrated_agent_evaluation/agentcore/online_version.py +15 -0
- calibrated_agent_evaluation/agentcore/optimization.py +525 -0
- calibrated_agent_evaluation/agentcore/policy_spans.py +466 -0
- calibrated_agent_evaluation/agentcore/pricing.py +349 -0
- calibrated_agent_evaluation/agentcore/readback.py +440 -0
- calibrated_agent_evaluation/agentcore/readers.py +795 -0
- calibrated_agent_evaluation/agentcore/replay.py +695 -0
- calibrated_agent_evaluation/agentcore/run_manifest.py +190 -0
- calibrated_agent_evaluation/agentcore/run_spans.py +106 -0
- calibrated_agent_evaluation/agentcore/runtime.py +1055 -0
- calibrated_agent_evaluation/agentcore/runtime_boundary.py +798 -0
- calibrated_agent_evaluation/agentcore/runtime_cassette.py +382 -0
- calibrated_agent_evaluation/agentcore/runtime_errors.py +49 -0
- calibrated_agent_evaluation/agentcore/runtime_executor.py +985 -0
- calibrated_agent_evaluation/agentcore/runtime_probe.py +306 -0
- calibrated_agent_evaluation/agentcore/runtime_response.py +307 -0
- calibrated_agent_evaluation/agentcore/scores.py +147 -0
- calibrated_agent_evaluation/agentcore/session.py +250 -0
- calibrated_agent_evaluation/agentcore/settle.py +96 -0
- calibrated_agent_evaluation/agentcore/simulated_user.py +139 -0
- calibrated_agent_evaluation/agentcore/sources.py +324 -0
- calibrated_agent_evaluation/agentcore/span_reader.py +399 -0
- calibrated_agent_evaluation/attribution/__init__.py +90 -0
- calibrated_agent_evaluation/attribution/agreement.py +107 -0
- calibrated_agent_evaluation/attribution/codebook.py +94 -0
- calibrated_agent_evaluation/attribution/definition.py +112 -0
- calibrated_agent_evaluation/attribution/errors.py +34 -0
- calibrated_agent_evaluation/attribution/labels.py +48 -0
- calibrated_agent_evaluation/attribution/rates.py +86 -0
- calibrated_agent_evaluation/attribution/relocation.py +139 -0
- calibrated_agent_evaluation/attribution/types.py +207 -0
- calibrated_agent_evaluation/attribution/walkback.py +190 -0
- calibrated_agent_evaluation/backends/__init__.py +79 -0
- calibrated_agent_evaluation/backends/_bounded.py +62 -0
- calibrated_agent_evaluation/backends/_export_tap.py +86 -0
- calibrated_agent_evaluation/backends/base.py +220 -0
- calibrated_agent_evaluation/backends/dashboard.py +334 -0
- calibrated_agent_evaluation/backends/factory.py +85 -0
- calibrated_agent_evaluation/backends/langfuse.py +1061 -0
- calibrated_agent_evaluation/backends/langfuse_foreign.py +210 -0
- calibrated_agent_evaluation/backends/local.py +322 -0
- calibrated_agent_evaluation/bias/__init__.py +80 -0
- calibrated_agent_evaluation/bias/battery.py +284 -0
- calibrated_agent_evaluation/bias/errors.py +131 -0
- calibrated_agent_evaluation/bias/gate.py +103 -0
- calibrated_agent_evaluation/bias/loader.py +120 -0
- calibrated_agent_evaluation/bias/mitigate.py +23 -0
- calibrated_agent_evaluation/bias/probe.py +170 -0
- calibrated_agent_evaluation/bias/record.py +123 -0
- calibrated_agent_evaluation/bias/store.py +62 -0
- calibrated_agent_evaluation/bias/thresholds.py +52 -0
- calibrated_agent_evaluation/bias/version.py +16 -0
- calibrated_agent_evaluation/calibrate/__init__.py +162 -0
- calibrated_agent_evaluation/calibrate/engine.py +570 -0
- calibrated_agent_evaluation/calibrate/errors.py +179 -0
- calibrated_agent_evaluation/calibrate/ledger.py +200 -0
- calibrated_agent_evaluation/calibrate/pairing.py +137 -0
- calibrated_agent_evaluation/calibrate/plan.py +411 -0
- calibrated_agent_evaluation/calibrate/record.py +282 -0
- calibrated_agent_evaluation/calibrate/store.py +67 -0
- calibrated_agent_evaluation/cli/__init__.py +31 -0
- calibrated_agent_evaluation/cli/agentcore_route.py +435 -0
- calibrated_agent_evaluation/cli/agentspec.py +43 -0
- calibrated_agent_evaluation/cli/args.py +1714 -0
- calibrated_agent_evaluation/cli/atoms.py +75 -0
- calibrated_agent_evaluation/cli/backends.py +168 -0
- calibrated_agent_evaluation/cli/config.py +146 -0
- calibrated_agent_evaluation/cli/consistency.py +461 -0
- calibrated_agent_evaluation/cli/conversation_route.py +107 -0
- calibrated_agent_evaluation/cli/dispatch.py +167 -0
- calibrated_agent_evaluation/cli/drift.py +206 -0
- calibrated_agent_evaluation/cli/errors.py +111 -0
- calibrated_agent_evaluation/cli/goldagreement.py +200 -0
- calibrated_agent_evaluation/cli/handlers.py +119 -0
- calibrated_agent_evaluation/cli/judgefile.py +233 -0
- calibrated_agent_evaluation/cli/judgekit.py +260 -0
- calibrated_agent_evaluation/cli/judging.py +213 -0
- calibrated_agent_evaluation/cli/pairing.py +324 -0
- calibrated_agent_evaluation/cli/parser.py +290 -0
- calibrated_agent_evaluation/cli/plain.py +558 -0
- calibrated_agent_evaluation/cli/present.py +104 -0
- calibrated_agent_evaluation/cli/proposals.py +58 -0
- calibrated_agent_evaluation/cli/records.py +655 -0
- calibrated_agent_evaluation/cli/scope.py +70 -0
- calibrated_agent_evaluation/cli/stages/__init__.py +9 -0
- calibrated_agent_evaluation/cli/stages/agreement.py +196 -0
- calibrated_agent_evaluation/cli/stages/audit.py +860 -0
- calibrated_agent_evaluation/cli/stages/bias.py +200 -0
- calibrated_agent_evaluation/cli/stages/calibrate.py +466 -0
- calibrated_agent_evaluation/cli/stages/calibrate_agentcore.py +311 -0
- calibrated_agent_evaluation/cli/stages/calibrate_check.py +269 -0
- calibrated_agent_evaluation/cli/stages/capture.py +52 -0
- calibrated_agent_evaluation/cli/stages/cluster.py +248 -0
- calibrated_agent_evaluation/cli/stages/codebook.py +110 -0
- calibrated_agent_evaluation/cli/stages/dataset.py +353 -0
- calibrated_agent_evaluation/cli/stages/dataset_run.py +481 -0
- calibrated_agent_evaluation/cli/stages/define.py +79 -0
- calibrated_agent_evaluation/cli/stages/doctor.py +259 -0
- calibrated_agent_evaluation/cli/stages/dumpload.py +186 -0
- calibrated_agent_evaluation/cli/stages/generate.py +679 -0
- calibrated_agent_evaluation/cli/stages/improve.py +1049 -0
- calibrated_agent_evaluation/cli/stages/ingest.py +434 -0
- calibrated_agent_evaluation/cli/stages/judge.py +336 -0
- calibrated_agent_evaluation/cli/stages/label.py +225 -0
- calibrated_agent_evaluation/cli/stages/label_pairwise.py +111 -0
- calibrated_agent_evaluation/cli/stages/online.py +359 -0
- calibrated_agent_evaluation/cli/stages/online_read.py +739 -0
- calibrated_agent_evaluation/cli/stages/plan.py +86 -0
- calibrated_agent_evaluation/cli/stages/prefer.py +232 -0
- calibrated_agent_evaluation/cli/stages/probe.py +656 -0
- calibrated_agent_evaluation/cli/stages/probe_policy.py +305 -0
- calibrated_agent_evaluation/cli/stages/quickstart.py +226 -0
- calibrated_agent_evaluation/cli/stages/ratify.py +131 -0
- calibrated_agent_evaluation/cli/stages/results.py +80 -0
- calibrated_agent_evaluation/cli/stages/run.py +851 -0
- calibrated_agent_evaluation/cli/stages/score.py +395 -0
- calibrated_agent_evaluation/cli/taxonomy.py +120 -0
- calibrated_agent_evaluation/cluster/__init__.py +49 -0
- calibrated_agent_evaluation/cluster/codebook.py +188 -0
- calibrated_agent_evaluation/cluster/coverage.py +64 -0
- calibrated_agent_evaluation/cluster/draft.py +73 -0
- calibrated_agent_evaluation/cluster/embed.py +40 -0
- calibrated_agent_evaluation/cluster/errors.py +16 -0
- calibrated_agent_evaluation/cluster/group.py +38 -0
- calibrated_agent_evaluation/cluster/ratify_map.py +302 -0
- calibrated_agent_evaluation/cluster/saturation.py +243 -0
- calibrated_agent_evaluation/cluster/types.py +87 -0
- calibrated_agent_evaluation/dumpload/__init__.py +76 -0
- calibrated_agent_evaluation/dumpload/io.py +576 -0
- calibrated_agent_evaluation/dumpload/registry.py +107 -0
- calibrated_agent_evaluation/dumpload/results.py +609 -0
- calibrated_agent_evaluation/enforce/__init__.py +107 -0
- calibrated_agent_evaluation/enforce/admission.py +130 -0
- calibrated_agent_evaluation/enforce/checklist.py +54 -0
- calibrated_agent_evaluation/enforce/context.py +28 -0
- calibrated_agent_evaluation/enforce/decision.py +67 -0
- calibrated_agent_evaluation/enforce/display.py +118 -0
- calibrated_agent_evaluation/enforce/errors.py +106 -0
- calibrated_agent_evaluation/enforce/gate.py +80 -0
- calibrated_agent_evaluation/enforce/judgedef.py +94 -0
- calibrated_agent_evaluation/enforce/lint.py +214 -0
- calibrated_agent_evaluation/enforce/store.py +101 -0
- calibrated_agent_evaluation/enforce/version.py +13 -0
- calibrated_agent_evaluation/execute/__init__.py +137 -0
- calibrated_agent_evaluation/execute/base.py +37 -0
- calibrated_agent_evaluation/execute/bedrock_embedder.py +123 -0
- calibrated_agent_evaluation/execute/bedrock_renderer.py +64 -0
- calibrated_agent_evaluation/execute/benchmark_tasks.py +50 -0
- calibrated_agent_evaluation/execute/cassette.py +459 -0
- calibrated_agent_evaluation/execute/cassette_provider.py +157 -0
- calibrated_agent_evaluation/execute/errors.py +43 -0
- calibrated_agent_evaluation/execute/inspect_executor.py +221 -0
- calibrated_agent_evaluation/execute/judge_model.py +459 -0
- calibrated_agent_evaluation/execute/probe_bridge.py +207 -0
- calibrated_agent_evaluation/execute/sandbox.py +274 -0
- calibrated_agent_evaluation/execute/strands_capture.py +323 -0
- calibrated_agent_evaluation/execute/strands_cassette_model.py +132 -0
- calibrated_agent_evaluation/execute/strands_executor.py +281 -0
- calibrated_agent_evaluation/execute/task_builders.py +142 -0
- calibrated_agent_evaluation/flagship/__init__.py +110 -0
- calibrated_agent_evaluation/flagship/errors.py +57 -0
- calibrated_agent_evaluation/flagship/oracle.py +112 -0
- calibrated_agent_evaluation/flagship/orchestrator.py +362 -0
- calibrated_agent_evaluation/flagship/reference.py +525 -0
- calibrated_agent_evaluation/flagship/report.py +199 -0
- calibrated_agent_evaluation/flagship/runner.py +183 -0
- calibrated_agent_evaluation/flagship/stages.py +292 -0
- calibrated_agent_evaluation/flagship/store.py +62 -0
- calibrated_agent_evaluation/improve/__init__.py +132 -0
- calibrated_agent_evaluation/improve/confirm.py +95 -0
- calibrated_agent_evaluation/improve/crew.py +344 -0
- calibrated_agent_evaluation/improve/errors.py +75 -0
- calibrated_agent_evaluation/improve/guard.py +244 -0
- calibrated_agent_evaluation/improve/paired.py +230 -0
- calibrated_agent_evaluation/improve/panel.py +196 -0
- calibrated_agent_evaluation/improve/prereg.py +263 -0
- calibrated_agent_evaluation/improve/report.py +311 -0
- calibrated_agent_evaluation/improve/store.py +161 -0
- calibrated_agent_evaluation/ingest/__init__.py +60 -0
- calibrated_agent_evaluation/ingest/core.py +1537 -0
- calibrated_agent_evaluation/ingest/detect.py +199 -0
- calibrated_agent_evaluation/ingest/errors.py +38 -0
- calibrated_agent_evaluation/ingest/sample.py +147 -0
- calibrated_agent_evaluation/judges/__init__.py +87 -0
- calibrated_agent_evaluation/judges/cache.py +140 -0
- calibrated_agent_evaluation/judges/errors.py +35 -0
- calibrated_agent_evaluation/judges/executable.py +92 -0
- calibrated_agent_evaluation/judges/levels.py +60 -0
- calibrated_agent_evaluation/judges/probe_runner.py +152 -0
- calibrated_agent_evaluation/judges/process.py +90 -0
- calibrated_agent_evaluation/judges/run_record.py +124 -0
- calibrated_agent_evaluation/judges/runtime.py +300 -0
- calibrated_agent_evaluation/judges/targets.py +163 -0
- calibrated_agent_evaluation/judges/types.py +56 -0
- calibrated_agent_evaluation/judges/verdict.py +83 -0
- calibrated_agent_evaluation/judges/verdict_schema.py +65 -0
- calibrated_agent_evaluation/judges/version.py +14 -0
- calibrated_agent_evaluation/normalize/__init__.py +185 -0
- calibrated_agent_evaluation/normalize/_depth.py +34 -0
- calibrated_agent_evaluation/normalize/agent_boundaries.py +65 -0
- calibrated_agent_evaluation/normalize/content.py +177 -0
- calibrated_agent_evaluation/normalize/content_placements.py +176 -0
- calibrated_agent_evaluation/normalize/conventions/__init__.py +41 -0
- calibrated_agent_evaluation/normalize/conventions/detect.py +121 -0
- calibrated_agent_evaluation/normalize/conventions/dispatch.py +31 -0
- calibrated_agent_evaluation/normalize/conventions/messages_adapter.py +269 -0
- calibrated_agent_evaluation/normalize/conventions/openinference_adapter.py +222 -0
- calibrated_agent_evaluation/normalize/conventions/openllmetry_adapter.py +260 -0
- calibrated_agent_evaluation/normalize/conventions/pairing.py +118 -0
- calibrated_agent_evaluation/normalize/conventions/registry.py +80 -0
- calibrated_agent_evaluation/normalize/datasets/__init__.py +154 -0
- calibrated_agent_evaluation/normalize/datasets/hf.py +78 -0
- calibrated_agent_evaluation/normalize/datasets/jsonl.py +44 -0
- calibrated_agent_evaluation/normalize/datasets/parquet.py +58 -0
- calibrated_agent_evaluation/normalize/errors.py +28 -0
- calibrated_agent_evaluation/normalize/guidance.py +70 -0
- calibrated_agent_evaluation/normalize/inspect_adapter.py +259 -0
- calibrated_agent_evaluation/normalize/otel_adapter.py +626 -0
- calibrated_agent_evaluation/normalize/otlp_json.py +363 -0
- calibrated_agent_evaluation/normalize/ownership.py +122 -0
- calibrated_agent_evaluation/normalize/sources.py +212 -0
- calibrated_agent_evaluation/normalize/strands_adapter.py +440 -0
- calibrated_agent_evaluation/normalize/tree.py +284 -0
- calibrated_agent_evaluation/probe/__init__.py +156 -0
- calibrated_agent_evaluation/probe/adapter.py +328 -0
- calibrated_agent_evaluation/probe/authorize.py +54 -0
- calibrated_agent_evaluation/probe/battery.py +180 -0
- calibrated_agent_evaluation/probe/benchmarks.py +214 -0
- calibrated_agent_evaluation/probe/capability.py +36 -0
- calibrated_agent_evaluation/probe/connection.py +139 -0
- calibrated_agent_evaluation/probe/errors.py +82 -0
- calibrated_agent_evaluation/probe/fingerprint.py +20 -0
- calibrated_agent_evaluation/probe/framing.py +29 -0
- calibrated_agent_evaluation/probe/harness.py +263 -0
- calibrated_agent_evaluation/probe/policy.py +561 -0
- calibrated_agent_evaluation/probe/policy_battery.py +296 -0
- calibrated_agent_evaluation/probe/rate.py +72 -0
- calibrated_agent_evaluation/probe/record.py +85 -0
- calibrated_agent_evaluation/probe/redact.py +32 -0
- calibrated_agent_evaluation/probe/robustness.py +174 -0
- calibrated_agent_evaluation/probe/store.py +61 -0
- calibrated_agent_evaluation/probe/version.py +8 -0
- calibrated_agent_evaluation/probe/ws_adapter.py +207 -0
- calibrated_agent_evaluation/py.typed +0 -0
- calibrated_agent_evaluation/review/__init__.py +150 -0
- calibrated_agent_evaluation/review/addresses.py +55 -0
- calibrated_agent_evaluation/review/agreement.py +255 -0
- calibrated_agent_evaluation/review/app.py +161 -0
- calibrated_agent_evaluation/review/context.py +254 -0
- calibrated_agent_evaluation/review/errors.py +45 -0
- calibrated_agent_evaluation/review/gate.py +36 -0
- calibrated_agent_evaluation/review/guards.py +152 -0
- calibrated_agent_evaluation/review/labeling.py +144 -0
- calibrated_agent_evaluation/review/overlap.py +241 -0
- calibrated_agent_evaluation/review/pairwise.py +339 -0
- calibrated_agent_evaluation/review/prefill.py +46 -0
- calibrated_agent_evaluation/review/proposals.py +43 -0
- calibrated_agent_evaluation/review/record.py +332 -0
- calibrated_agent_evaluation/review/relabel.py +36 -0
- calibrated_agent_evaluation/review/renderers/__init__.py +49 -0
- calibrated_agent_evaluation/review/renderers/base.py +97 -0
- calibrated_agent_evaluation/review/renderers/default.py +107 -0
- calibrated_agent_evaluation/review/renderers/trace_view.py +358 -0
- calibrated_agent_evaluation/review/routes.py +465 -0
- calibrated_agent_evaluation/review/session.py +269 -0
- calibrated_agent_evaluation/review/session_log.py +66 -0
- calibrated_agent_evaluation/review/static/hotkeys.js +385 -0
- calibrated_agent_evaluation/review/static/pairwise.js +28 -0
- calibrated_agent_evaluation/review/static/styles.css +130 -0
- calibrated_agent_evaluation/review/static/vendor/README.md +13 -0
- calibrated_agent_evaluation/review/static/vendor/htmx.min.js +1 -0
- calibrated_agent_evaluation/review/store.py +99 -0
- calibrated_agent_evaluation/run/__init__.py +143 -0
- calibrated_agent_evaluation/run/aggregate.py +75 -0
- calibrated_agent_evaluation/run/drift_report.py +333 -0
- calibrated_agent_evaluation/run/errors.py +54 -0
- calibrated_agent_evaluation/run/features.py +88 -0
- calibrated_agent_evaluation/run/flags.py +40 -0
- calibrated_agent_evaluation/run/gate.py +210 -0
- calibrated_agent_evaluation/run/judge_check.py +161 -0
- calibrated_agent_evaluation/run/multi.py +41 -0
- calibrated_agent_evaluation/run/report.py +457 -0
- calibrated_agent_evaluation/run/staleness.py +81 -0
- calibrated_agent_evaluation/run/store.py +61 -0
- calibrated_agent_evaluation/run/trend.py +161 -0
- calibrated_agent_evaluation/sampling/__init__.py +49 -0
- calibrated_agent_evaluation/sampling/discovery.py +161 -0
- calibrated_agent_evaluation/sampling/errors.py +26 -0
- calibrated_agent_evaluation/sampling/production.py +192 -0
- calibrated_agent_evaluation/sampling/stats.py +44 -0
- calibrated_agent_evaluation/sampling/types.py +48 -0
- calibrated_agent_evaluation/sampling/validate.py +53 -0
- calibrated_agent_evaluation/scenarios/__init__.py +56 -0
- calibrated_agent_evaluation/scenarios/build.py +224 -0
- calibrated_agent_evaluation/scenarios/errors.py +14 -0
- calibrated_agent_evaluation/scenarios/labels.py +49 -0
- calibrated_agent_evaluation/scenarios/slicefile.py +101 -0
- calibrated_agent_evaluation/scenarios/types.py +212 -0
- calibrated_agent_evaluation/schema/__init__.py +70 -0
- calibrated_agent_evaluation/schema/errors.py +58 -0
- calibrated_agent_evaluation/schema/serde.py +182 -0
- calibrated_agent_evaluation/schema/types.py +174 -0
- calibrated_agent_evaluation/schema/validate.py +213 -0
- calibrated_agent_evaluation/scorers/__init__.py +86 -0
- calibrated_agent_evaluation/scorers/base.py +47 -0
- calibrated_agent_evaluation/scorers/batch.py +51 -0
- calibrated_agent_evaluation/scorers/errors.py +25 -0
- calibrated_agent_evaluation/scorers/execution.py +475 -0
- calibrated_agent_evaluation/scorers/heuristic.py +312 -0
- calibrated_agent_evaluation/scorers/lane.py +28 -0
- calibrated_agent_evaluation/scorers/passk.py +129 -0
- calibrated_agent_evaluation/stats/__init__.py +55 -0
- calibrated_agent_evaluation/stats/agreement.py +311 -0
- calibrated_agent_evaluation/stats/drift.py +353 -0
- calibrated_agent_evaluation/stats/kappa.py +106 -0
- calibrated_agent_evaluation/stats/mcnemar.py +178 -0
- calibrated_agent_evaluation/stats/multiplicity.py +60 -0
- calibrated_agent_evaluation/stats/proportions.py +137 -0
- calibrated_agent_evaluation/stats/rogan_gladen.py +342 -0
- calibrated_agent_evaluation/stats/studies/__init__.py +19 -0
- calibrated_agent_evaluation/stats/studies/baseline_arm.py +143 -0
- calibrated_agent_evaluation/stats/studies/coverage.py +325 -0
- calibrated_agent_evaluation/stats/studies/planner_floor.py +160 -0
- calibrated_agent_evaluation/stats/studies/rate_sample_minimum.py +147 -0
- calibrated_agent_evaluation/subjective/__init__.py +70 -0
- calibrated_agent_evaluation/subjective/atomfile.py +86 -0
- calibrated_agent_evaluation/subjective/bradley_terry.py +192 -0
- calibrated_agent_evaluation/subjective/classify.py +68 -0
- calibrated_agent_evaluation/subjective/elo.py +96 -0
- calibrated_agent_evaluation/subjective/errors.py +20 -0
- calibrated_agent_evaluation/subjective/gate.py +29 -0
- calibrated_agent_evaluation/subjective/pairfile.py +91 -0
- calibrated_agent_evaluation/subjective/preference.py +49 -0
- calibrated_agent_evaluation/subjective/store.py +142 -0
- calibrated_agent_evaluation/subjective/types.py +99 -0
- calibrated_agent_evaluation/synth/__init__.py +50 -0
- calibrated_agent_evaluation/synth/borrow/__init__.py +13 -0
- calibrated_agent_evaluation/synth/borrow/simulator.py +247 -0
- calibrated_agent_evaluation/synth/conversation.py +460 -0
- calibrated_agent_evaluation/synth/conversation_cassette.py +248 -0
- calibrated_agent_evaluation/synth/errors.py +83 -0
- calibrated_agent_evaluation/synth/generate.py +352 -0
- calibrated_agent_evaluation/synth/guards.py +115 -0
- calibrated_agent_evaluation/synth/risk.py +75 -0
- calibrated_agent_evaluation/synth/types.py +78 -0
- calibrated_agent_evaluation-0.1.0.dist-info/METADATA +237 -0
- calibrated_agent_evaluation-0.1.0.dist-info/RECORD +416 -0
- calibrated_agent_evaluation-0.1.0.dist-info/WHEEL +4 -0
- calibrated_agent_evaluation-0.1.0.dist-info/entry_points.txt +2 -0
- calibrated_agent_evaluation-0.1.0.dist-info/licenses/LICENSE +175 -0
- calibrated_agent_evaluation-0.1.0.dist-info/licenses/NOTICE +2 -0
- calibrated_agent_evaluation-0.1.0.dist-info/licenses/THIRD-PARTY-LICENSES +446 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Calibrated Agent Evaluation: error analysis and judge calibration for AI agents.
|
|
5
|
+
|
|
6
|
+
The distribution and the command-line entry point are both ``calibrated-agent-evaluation``.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
__version__ = version("calibrated-agent-evaluation")
|
|
13
|
+
except PackageNotFoundError: # pragma: no cover - package not installed
|
|
14
|
+
__version__ = "0.0.0+unknown"
|
|
15
|
+
|
|
16
|
+
__all__ = ["__version__"]
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""``python -m calibrated_agent_evaluation``: the ``calibrated-agent-evaluation`` command."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from calibrated_agent_evaluation.cli import main
|
|
9
|
+
|
|
10
|
+
if __name__ == "__main__":
|
|
11
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""The value types both sides of a simulated conversation share, kept outside the ``synth`` plane.
|
|
5
|
+
|
|
6
|
+
The AgentCore adapters need these types at run time (the Runtime executor checks a task's
|
|
7
|
+
:class:`ConversationTurn`, the AgentCore simulated user returns a :class:`UserTurn`), and the
|
|
8
|
+
``[agentcore]`` extra does not install the sampling stack that importing anything under
|
|
9
|
+
``calibrated_agent_evaluation.synth`` loads. So the types live here, and
|
|
10
|
+
:mod:`calibrated_agent_evaluation.synth.conversation` re-exports them under its own names.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from typing import Literal, Protocol
|
|
17
|
+
|
|
18
|
+
from calibrated_agent_evaluation._model import CalibratedAgentEvaluationModel
|
|
19
|
+
|
|
20
|
+
__all__ = ["ConversationTurn", "SimulatedUser", "StopReason", "UserTurn"]
|
|
21
|
+
|
|
22
|
+
StopReason = Literal["goal_completed", "actor_stopped", "max_turns"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class UserTurn(CalibratedAgentEvaluationModel):
|
|
26
|
+
"""The simulated user's next move: a message, or a stop with its reason.
|
|
27
|
+
|
|
28
|
+
A stop may carry a last message (a customer who says "Thanks, that's all" and leaves); the
|
|
29
|
+
conversation keeps it, and no agent turn follows it. A turn with ``stop=False`` but no message
|
|
30
|
+
is read as a stop (``actor_stopped``): there is no customer message to send, and inventing one
|
|
31
|
+
would put a turn in the transcript nobody took.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
message: str | None
|
|
35
|
+
stop: bool
|
|
36
|
+
stop_reason: StopReason | None = None
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class SimulatedUser(Protocol):
|
|
40
|
+
"""The port a simulated-user adapter implements."""
|
|
41
|
+
|
|
42
|
+
def act(self, agent_message: str) -> UserTurn:
|
|
43
|
+
"""Answer the agent's latest reply with the next customer move."""
|
|
44
|
+
...
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class ConversationTurn:
|
|
49
|
+
"""Which conversation and which agent turn (1-based) a task belongs to."""
|
|
50
|
+
|
|
51
|
+
conversation_id: str
|
|
52
|
+
turn: int
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""The root of the library's typed error hierarchy.
|
|
5
|
+
|
|
6
|
+
It lives outside :mod:`calibrated_agent_evaluation.schema` so that the model base in
|
|
7
|
+
:mod:`calibrated_agent_evaluation._model`, which the schema types themselves build on, can name it
|
|
8
|
+
without importing the schema package.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from typing import TYPE_CHECKING
|
|
14
|
+
|
|
15
|
+
if TYPE_CHECKING:
|
|
16
|
+
from calibrated_agent_evaluation._install import InstallSource
|
|
17
|
+
|
|
18
|
+
__all__ = ["CalibratedAgentEvaluationDataError", "MissingExtraError"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class CalibratedAgentEvaluationDataError(Exception):
|
|
22
|
+
"""Base class for every data-plane error."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class MissingExtraError(CalibratedAgentEvaluationDataError):
|
|
26
|
+
"""A capability was asked for whose optional extra is not installed.
|
|
27
|
+
|
|
28
|
+
``hint`` is the one command that installs ``extras`` into this installation, matched to how
|
|
29
|
+
Calibrated Agent Evaluation was installed (checkout, editable, uv tool, wheel or index).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self, extras: tuple[str, ...], purpose: str, *, source: InstallSource | None = None
|
|
34
|
+
) -> None:
|
|
35
|
+
# Imported here: _install builds on pydantic, and this module must stay importable first.
|
|
36
|
+
from calibrated_agent_evaluation._install import _running_source, hint_for
|
|
37
|
+
|
|
38
|
+
self.extras = extras
|
|
39
|
+
self.hint = hint_for(source or _running_source(), extras)
|
|
40
|
+
names = ", ".join(f"[{e}]" for e in extras)
|
|
41
|
+
super().__init__(f"{purpose} needs the {names} extra; install it with: {self.hint}")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
# Public home is calibrated_agent_evaluation.schema.errors, which re-exports them; tracebacks and
|
|
45
|
+
# the reference say so.
|
|
46
|
+
CalibratedAgentEvaluationDataError.__module__ = "calibrated_agent_evaluation.schema.errors"
|
|
47
|
+
MissingExtraError.__module__ = "calibrated_agent_evaluation.schema.errors"
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""An advisory, cross-process exclusive lock on a sidecar file.
|
|
5
|
+
|
|
6
|
+
``fcntl.flock`` on POSIX, ``msvcrt.locking`` on Windows. Advisory: it serializes cooperating
|
|
7
|
+
writers (two ``calibrated-agent-evaluation`` processes recording one cassette), not arbitrary
|
|
8
|
+
programs. Not reentrant.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import os
|
|
14
|
+
from collections.abc import Iterator
|
|
15
|
+
from contextlib import contextmanager
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
|
|
18
|
+
__all__ = ["exclusive_lock"]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@contextmanager
|
|
22
|
+
def exclusive_lock(lock_path: Path) -> Iterator[None]:
|
|
23
|
+
"""Hold an exclusive lock on ``lock_path`` (created if absent) for the ``with`` block.
|
|
24
|
+
|
|
25
|
+
Raises:
|
|
26
|
+
OSError: The lock file cannot be created or locked.
|
|
27
|
+
"""
|
|
28
|
+
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
|
29
|
+
with lock_path.open("a+b") as fh:
|
|
30
|
+
_lock(fh.fileno())
|
|
31
|
+
try:
|
|
32
|
+
yield
|
|
33
|
+
finally:
|
|
34
|
+
_unlock(fh.fileno())
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
if os.name == "nt": # pragma: no cover - exercised on Windows only
|
|
38
|
+
import msvcrt
|
|
39
|
+
|
|
40
|
+
def _lock(fd: int) -> None:
|
|
41
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
42
|
+
msvcrt.locking(fd, msvcrt.LK_LOCK, 1)
|
|
43
|
+
|
|
44
|
+
def _unlock(fd: int) -> None:
|
|
45
|
+
os.lseek(fd, 0, os.SEEK_SET)
|
|
46
|
+
msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
|
|
47
|
+
|
|
48
|
+
else:
|
|
49
|
+
import fcntl
|
|
50
|
+
|
|
51
|
+
def _lock(fd: int) -> None:
|
|
52
|
+
fcntl.flock(fd, fcntl.LOCK_EX)
|
|
53
|
+
|
|
54
|
+
def _unlock(fd: int) -> None:
|
|
55
|
+
fcntl.flock(fd, fcntl.LOCK_UN)
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""Refuse a value that standard JSON cannot represent, by one rule on every supported Python.
|
|
5
|
+
|
|
6
|
+
Two things make a value one that JSON cannot represent here:
|
|
7
|
+
|
|
8
|
+
- **A NaN or infinite number.** Python's ``json`` writes and reads the ``NaN``, ``Infinity`` and
|
|
9
|
+
``-Infinity`` tokens unless told not to, but they are not JSON. A literal too large for a float,
|
|
10
|
+
such as ``1e999``, decodes to an infinity with no token, so a decoded value is checked too.
|
|
11
|
+
- **Nesting deeper than** :data:`MAX_JSON_DEPTH` **levels.** How deep Python's encoder and decoder
|
|
12
|
+
go before ``RecursionError`` differs between versions, so a fixed limit, checked before any
|
|
13
|
+
encoder runs, gives one answer everywhere.
|
|
14
|
+
|
|
15
|
+
:func:`require_finite` checks a value for both, and :func:`refuse_json_constant` is the
|
|
16
|
+
``parse_constant`` hook that refuses the three tokens during a decode. Both raise ``ValueError``
|
|
17
|
+
subclasses, so a caller's existing "not JSON" handling catches them. Their messages name where the
|
|
18
|
+
value came from and the rule it broke, never a key or a value of it.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import math
|
|
24
|
+
from collections.abc import Mapping
|
|
25
|
+
from typing import Final, NoReturn
|
|
26
|
+
|
|
27
|
+
__all__ = [
|
|
28
|
+
"MAX_JSON_DEPTH",
|
|
29
|
+
"NestedTooDeepError",
|
|
30
|
+
"NonFiniteNumberError",
|
|
31
|
+
"refuse_json_constant",
|
|
32
|
+
"require_finite",
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
MAX_JSON_DEPTH: Final = 256
|
|
36
|
+
"""The deepest nesting a value may have.
|
|
37
|
+
|
|
38
|
+
A value's depth is the number of containers (mappings, lists, tuples, sets and frozensets) on its
|
|
39
|
+
deepest path, the outermost included: a scalar has depth 0, ``[]`` and ``{"a": 1}`` have depth 1,
|
|
40
|
+
and ``[[1]]`` has depth 2. A mapping's keys count as items of the mapping. Each caller counts from
|
|
41
|
+
the value it checks.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class NonFiniteNumberError(ValueError):
|
|
46
|
+
"""A value holds a NaN or infinite number, which standard JSON has no form for."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class NestedTooDeepError(ValueError):
|
|
50
|
+
"""A value nests more than :data:`MAX_JSON_DEPTH` levels deep."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def require_finite(value: object, *, where: str) -> None:
|
|
54
|
+
"""Refuse ``value`` if it holds a NaN or infinite number, or nests too deep.
|
|
55
|
+
|
|
56
|
+
Walks mapping keys and values and the items of lists, tuples, sets and frozensets, stopping at
|
|
57
|
+
the first problem. It keeps its own stack and never looks inside a container past the limit,
|
|
58
|
+
so it cannot raise ``RecursionError``. Any other type is left to the caller's encoder.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
value: The value to check, its depth counted from here.
|
|
62
|
+
where: What the value is, such as ``"request body"``; the message names it.
|
|
63
|
+
|
|
64
|
+
Raises:
|
|
65
|
+
NonFiniteNumberError: A ``float`` in the value is NaN or infinite.
|
|
66
|
+
NestedTooDeepError: A container in the value sits deeper than :data:`MAX_JSON_DEPTH`
|
|
67
|
+
levels.
|
|
68
|
+
"""
|
|
69
|
+
# Each item with the level of the container holding it: 0 for the value itself.
|
|
70
|
+
pending: list[tuple[object, int]] = [(value, 0)]
|
|
71
|
+
while pending:
|
|
72
|
+
item, holder_level = pending.pop()
|
|
73
|
+
if isinstance(item, float):
|
|
74
|
+
if not math.isfinite(item):
|
|
75
|
+
raise NonFiniteNumberError(
|
|
76
|
+
f"{where} holds a number JSON cannot represent ({_kind(item)})"
|
|
77
|
+
)
|
|
78
|
+
continue
|
|
79
|
+
children = _children(item)
|
|
80
|
+
if children is None:
|
|
81
|
+
continue
|
|
82
|
+
level = holder_level + 1
|
|
83
|
+
if level > MAX_JSON_DEPTH:
|
|
84
|
+
raise NestedTooDeepError(f"{where} nests more than {MAX_JSON_DEPTH} levels deep")
|
|
85
|
+
# Reversed, so items are met in the order the value holds them.
|
|
86
|
+
pending.extend((child, level) for child in reversed(children))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def refuse_json_constant(token: str) -> NoReturn:
|
|
90
|
+
"""The ``parse_constant`` hook of a standard-JSON decode: refuse every constant it is given.
|
|
91
|
+
|
|
92
|
+
``json.loads`` calls it only for the ``NaN``, ``Infinity`` and ``-Infinity`` tokens.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
token: The token the decoder met.
|
|
96
|
+
|
|
97
|
+
Raises:
|
|
98
|
+
NonFiniteNumberError: Always, naming the token.
|
|
99
|
+
"""
|
|
100
|
+
# Python spells the three tokens the way float() reads them.
|
|
101
|
+
raise NonFiniteNumberError(
|
|
102
|
+
f"the JSON text holds the token {token}, a number JSON cannot represent "
|
|
103
|
+
f"({_kind(float(token))})"
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _kind(number: float) -> str:
|
|
108
|
+
return "NaN" if math.isnan(number) else "infinite"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _children(item: object) -> list[object] | None:
|
|
112
|
+
# A container's items, keys included; None for anything that is not a container.
|
|
113
|
+
if isinstance(item, Mapping):
|
|
114
|
+
children: list[object] = []
|
|
115
|
+
for key, child in item.items():
|
|
116
|
+
children.append(key)
|
|
117
|
+
children.append(child)
|
|
118
|
+
return children
|
|
119
|
+
if isinstance(item, list | tuple | set | frozenset):
|
|
120
|
+
return list(item)
|
|
121
|
+
return None
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""The command that adds an optional extra to the running installation.
|
|
5
|
+
|
|
6
|
+
Every "this needs an extra" message renders its install command here, so the command matches how
|
|
7
|
+
the package was installed: the checkout's ``uv sync``, a ``uv tool`` reinstall, or an install into
|
|
8
|
+
the running interpreter's environment, with pip when pip installed it and uv otherwise. Arguments
|
|
9
|
+
are quoted for the platform's shell.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import shlex
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import tomllib
|
|
20
|
+
from functools import cache
|
|
21
|
+
from importlib import metadata
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Literal, Protocol
|
|
24
|
+
from urllib.parse import urlparse
|
|
25
|
+
from urllib.request import url2pathname
|
|
26
|
+
|
|
27
|
+
from pydantic import BaseModel, ConfigDict
|
|
28
|
+
|
|
29
|
+
__all__ = ["InstallSource", "detect_install_source", "hint_for", "install_hint"]
|
|
30
|
+
|
|
31
|
+
_DISTRIBUTION = "calibrated-agent-evaluation"
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class _Distribution(Protocol):
|
|
35
|
+
def read_text(self, filename: str) -> str | None: ...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class InstallSource(BaseModel):
|
|
39
|
+
"""How the running Calibrated Agent Evaluation was installed."""
|
|
40
|
+
|
|
41
|
+
model_config = ConfigDict(frozen=True)
|
|
42
|
+
|
|
43
|
+
kind: Literal["project", "editable", "tool", "local", "index"]
|
|
44
|
+
path: str | None = None
|
|
45
|
+
"""The wheel or directory a ``tool`` or ``local`` install came from."""
|
|
46
|
+
tool_extras: tuple[str, ...] = ()
|
|
47
|
+
"""Extras already in a ``uv tool`` environment, which a reinstall must repeat to keep."""
|
|
48
|
+
installer: str | None = None
|
|
49
|
+
"""The tool the distribution's ``INSTALLER`` file names (``pip``, ``uv``), when it has one."""
|
|
50
|
+
python: str | None = None
|
|
51
|
+
"""The interpreter of the environment, which an install command has to target."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _direct_url(dist: _Distribution) -> dict[str, object]:
|
|
55
|
+
raw = dist.read_text("direct_url.json")
|
|
56
|
+
if raw is None:
|
|
57
|
+
return {}
|
|
58
|
+
try:
|
|
59
|
+
parsed = json.loads(raw)
|
|
60
|
+
except json.JSONDecodeError:
|
|
61
|
+
return {}
|
|
62
|
+
return parsed if isinstance(parsed, dict) else {}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _tool_receipt(prefix: Path) -> tuple[tuple[str, ...], str | None] | None:
|
|
66
|
+
"""The extras and source path recorded in a ``uv tool`` environment, or None outside one."""
|
|
67
|
+
receipt = prefix / "uv-receipt.toml"
|
|
68
|
+
if not receipt.is_file():
|
|
69
|
+
return None
|
|
70
|
+
try:
|
|
71
|
+
doc = tomllib.loads(receipt.read_text(encoding="utf-8"))
|
|
72
|
+
except (OSError, tomllib.TOMLDecodeError):
|
|
73
|
+
return (), None
|
|
74
|
+
for req in doc.get("tool", {}).get("requirements", []):
|
|
75
|
+
if isinstance(req, dict) and req.get("name") == _DISTRIBUTION:
|
|
76
|
+
return tuple(req.get("extras", ())), req.get("path")
|
|
77
|
+
return (), None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def detect_install_source(
|
|
81
|
+
dist: _Distribution | None, prefix: Path, python: str | None = None
|
|
82
|
+
) -> InstallSource:
|
|
83
|
+
"""Classify an installation from its distribution metadata and environment prefix.
|
|
84
|
+
|
|
85
|
+
Args:
|
|
86
|
+
dist: The installed ``calibrated-agent-evaluation`` distribution, or None when it was
|
|
87
|
+
imported from a source tree that was never installed.
|
|
88
|
+
prefix: The environment's ``sys.prefix``.
|
|
89
|
+
python: The environment's interpreter, which the install command targets.
|
|
90
|
+
"""
|
|
91
|
+
if dist is None:
|
|
92
|
+
return InstallSource(kind="editable", python=python)
|
|
93
|
+
installer = (dist.read_text("INSTALLER") or "").strip().lower() or None
|
|
94
|
+
source = _classify(dist, prefix)
|
|
95
|
+
return source.model_copy(update={"installer": installer, "python": python})
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _classify(dist: _Distribution, prefix: Path) -> InstallSource:
|
|
99
|
+
direct = _direct_url(dist)
|
|
100
|
+
url = str(direct.get("url", ""))
|
|
101
|
+
# A file URL is percent-encoded: a checkout path with a space arrives as `%20`.
|
|
102
|
+
local_path = url2pathname(urlparse(url).path) if url.startswith("file://") else None
|
|
103
|
+
dir_info = direct.get("dir_info")
|
|
104
|
+
if isinstance(dir_info, dict) and dir_info.get("editable"):
|
|
105
|
+
# `uv sync` installs the checkout into <checkout>/.venv; any other editable install was
|
|
106
|
+
# made into some other environment by hand.
|
|
107
|
+
if local_path and prefix.resolve() == (Path(local_path) / ".venv").resolve():
|
|
108
|
+
return InstallSource(kind="project")
|
|
109
|
+
return InstallSource(kind="editable")
|
|
110
|
+
receipt = _tool_receipt(prefix)
|
|
111
|
+
if receipt is not None:
|
|
112
|
+
extras, receipt_path = receipt
|
|
113
|
+
return InstallSource(kind="tool", path=receipt_path or local_path, tool_extras=extras)
|
|
114
|
+
if local_path:
|
|
115
|
+
return InstallSource(kind="local", path=local_path)
|
|
116
|
+
return InstallSource(kind="index")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _quote(arg: str) -> str:
|
|
120
|
+
"""``arg`` as one word of the platform's shell."""
|
|
121
|
+
return subprocess.list2cmdline([arg]) if os.name == "nt" else shlex.quote(arg)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def hint_for(source: InstallSource, extras: tuple[str, ...]) -> str:
|
|
125
|
+
"""The command that adds ``extras`` to an installation of the given kind.
|
|
126
|
+
|
|
127
|
+
``uv sync`` removes packages it was not asked for, so the project form passes ``--inexact``.
|
|
128
|
+
``uv tool install`` replaces the tool's extras, so the tool form repeats the ones it has.
|
|
129
|
+
A path with extras has to be written as a direct reference (``name[x] @ path``); uv rejects
|
|
130
|
+
``path[x]``. Any other install is into the running interpreter's environment: a bare
|
|
131
|
+
``uv pip install`` targets whichever environment uv finds first, so the uv form names the
|
|
132
|
+
interpreter with ``--python``, and an install pip made gets ``<python> -m pip install``.
|
|
133
|
+
"""
|
|
134
|
+
names = ",".join(extras)
|
|
135
|
+
if source.kind == "project":
|
|
136
|
+
return "uv sync --inexact " + " ".join(f"--extra {e}" for e in extras)
|
|
137
|
+
if source.kind == "tool":
|
|
138
|
+
merged = ",".join(dict.fromkeys((*source.tool_extras, *extras)))
|
|
139
|
+
spec = f"{_DISTRIBUTION}[{merged}]"
|
|
140
|
+
return "uv tool install " + _quote(f"{spec} @ {source.path}" if source.path else spec)
|
|
141
|
+
if source.kind == "editable":
|
|
142
|
+
target = ["-e", _quote(f".[{names}]")]
|
|
143
|
+
elif source.kind == "local" and source.path:
|
|
144
|
+
target = [_quote(f"{_DISTRIBUTION}[{names}] @ {source.path}")]
|
|
145
|
+
else:
|
|
146
|
+
target = [_quote(f"{_DISTRIBUTION}[{names}]")]
|
|
147
|
+
if source.installer == "pip":
|
|
148
|
+
return " ".join([_quote(source.python or "python"), "-m", "pip", "install", *target])
|
|
149
|
+
python = ["--python", _quote(source.python)] if source.python else []
|
|
150
|
+
return " ".join(["uv", "pip", "install", *python, *target])
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@cache
|
|
154
|
+
def _running_source() -> InstallSource:
|
|
155
|
+
try:
|
|
156
|
+
dist: _Distribution | None = metadata.distribution(_DISTRIBUTION)
|
|
157
|
+
except metadata.PackageNotFoundError:
|
|
158
|
+
dist = None
|
|
159
|
+
return detect_install_source(dist, Path(sys.prefix), sys.executable)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def install_hint(*extras: str) -> str:
|
|
163
|
+
"""The shell command that installs ``extras`` into this installation.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
extras: The optional extras to add, by their ``pyproject`` names.
|
|
167
|
+
|
|
168
|
+
Returns:
|
|
169
|
+
One command: ``uv sync --inexact --extra X`` in the checkout's own environment,
|
|
170
|
+
``uv tool install`` for a uv tool, and otherwise ``uv pip install --python PYTHON`` (or
|
|
171
|
+
``PYTHON -m pip install`` when pip installed it) with ``-e '.[X]'`` for another editable
|
|
172
|
+
install or the distribution, as a direct reference to a local wheel or from an index.
|
|
173
|
+
"""
|
|
174
|
+
return hint_for(_running_source(), extras)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
|
|
4
|
+
"""The base for the library's pydantic models, and a validated copy-with-changes.
|
|
5
|
+
|
|
6
|
+
Every record and wire type subclasses :class:`CalibratedAgentEvaluationModel`, which turns a
|
|
7
|
+
pydantic ``ValidationError`` raised while building an instance into the model family's own typed
|
|
8
|
+
error (``invalid_error``), so a caller catches ``CalibrationError`` or ``RunError`` and never a raw
|
|
9
|
+
pydantic exception.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import dataclasses
|
|
15
|
+
from typing import Any, ClassVar, Self, TypeVar
|
|
16
|
+
|
|
17
|
+
from pydantic import (
|
|
18
|
+
BaseModel,
|
|
19
|
+
ConfigDict,
|
|
20
|
+
ModelWrapValidatorHandler,
|
|
21
|
+
ValidationError,
|
|
22
|
+
model_validator,
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
from calibrated_agent_evaluation._errors import CalibratedAgentEvaluationDataError
|
|
26
|
+
|
|
27
|
+
__all__ = ["CalibratedAgentEvaluationModel", "plain_error", "replace"]
|
|
28
|
+
|
|
29
|
+
T = TypeVar("T")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def plain_error(exc: ValidationError) -> str:
|
|
33
|
+
"""Summarize a validation failure as ``field: problem`` pairs, without pydantic's URLs."""
|
|
34
|
+
parts = []
|
|
35
|
+
for err in exc.errors(include_url=False):
|
|
36
|
+
where = ".".join(str(p) for p in err["loc"]) or "value"
|
|
37
|
+
parts.append(f"{where}: {err['msg']}")
|
|
38
|
+
return "; ".join(parts)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class CalibratedAgentEvaluationModel(BaseModel):
|
|
42
|
+
"""A pydantic model whose invalid input raises the family's typed error.
|
|
43
|
+
|
|
44
|
+
Subclasses set ``invalid_error`` to their package's base error and declare their own
|
|
45
|
+
``model_config`` (frozen, ``extra="forbid"``).
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# Building every model's schema at import costs each CLI command ~70 ms before it does
|
|
49
|
+
# anything; deferring builds a schema the first time its model is used.
|
|
50
|
+
model_config = ConfigDict(defer_build=True)
|
|
51
|
+
|
|
52
|
+
invalid_error: ClassVar[type[Exception]] = CalibratedAgentEvaluationDataError
|
|
53
|
+
|
|
54
|
+
@model_validator(mode="wrap")
|
|
55
|
+
@classmethod
|
|
56
|
+
def _raise_typed(cls, data: Any, handler: ModelWrapValidatorHandler[Self]) -> Self:
|
|
57
|
+
try:
|
|
58
|
+
return handler(data)
|
|
59
|
+
except ValidationError as exc:
|
|
60
|
+
raise cls.invalid_error(f"invalid {cls.__name__}: {plain_error(exc)}") from exc
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def replace(obj: T, /, **changes: Any) -> T:
|
|
64
|
+
"""Return a copy of ``obj`` with ``changes`` applied, validated like a fresh construction.
|
|
65
|
+
|
|
66
|
+
Unlike ``BaseModel.model_copy(update=...)``, the copy goes through the model's validators, so
|
|
67
|
+
a change that breaks an invariant raises exactly as constructing it would.
|
|
68
|
+
|
|
69
|
+
Args:
|
|
70
|
+
obj: A pydantic model or a dataclass instance.
|
|
71
|
+
**changes: Field values to replace.
|
|
72
|
+
|
|
73
|
+
Returns:
|
|
74
|
+
The new instance.
|
|
75
|
+
|
|
76
|
+
Raises:
|
|
77
|
+
TypeError: ``obj`` is neither a pydantic model nor a dataclass instance.
|
|
78
|
+
"""
|
|
79
|
+
if isinstance(obj, BaseModel):
|
|
80
|
+
fields = {name: getattr(obj, name) for name in type(obj).model_fields}
|
|
81
|
+
return type(obj)(**{**fields, **changes})
|
|
82
|
+
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
|
|
83
|
+
return dataclasses.replace(obj, **changes)
|
|
84
|
+
raise TypeError(f"replace() needs a pydantic model or a dataclass, got {type(obj).__name__}")
|