cmpnd 0.23.0__tar.gz → 0.23.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cmpnd-0.23.0 → cmpnd-0.23.2}/PKG-INFO +1 -1
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/callback.py +260 -46
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/login.py +3 -3
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/top.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/shell.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/eval_handler.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/models.py +89 -1
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/packaging.py +5 -5
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/sdk-instrumentation.md +28 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/pyproject.toml +1 -1
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/test-in-linux.sh +6 -6
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_assessments_cli.py +4 -4
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_optimize_review_committee.py +4 -6
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_valsets_cli.py +11 -14
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/optimization_run_probe.py +3 -3
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_assessments.py +9 -14
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_auth.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_datasets.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_deployments.py +14 -15
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_evals.py +3 -3
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_optimizations.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_parity.py +21 -22
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_programs.py +4 -4
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_schema_sample.py +7 -7
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_shell.py +82 -82
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_signatures.py +5 -5
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_traces.py +16 -17
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_traces_search_wire.py +18 -17
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_valsets.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cmp551_grouping_identity.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_deploy_mcp_tool_references.py +6 -7
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_exporter.py +84 -20
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_ir_encoder_properties.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_lm_usage_shapes.py +225 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_mcp_tools.py +66 -64
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimization_progress.py +11 -11
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize_terminal_failure.py +2 -2
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_packaging_mcp_tools.py +4 -4
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_reflector_properties.py +4 -4
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_reflector_smoke.py +3 -3
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_reversible_failures.py +3 -4
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_traces_read.py +9 -10
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_xxh64.py +3 -3
- {cmpnd-0.23.0 → cmpnd-0.23.2}/uv.lock +1 -1
- {cmpnd-0.23.0 → cmpnd-0.23.2}/.gitignore +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/CLAUDE.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/README.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/_exporter_pool.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/_gepa_patch.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/_http.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/_program_patch.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/_rlm_patch.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/_timespec.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/api_client.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/admin.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/assessments.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/auth.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/datasets.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/deployments.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/evals.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/optimizations.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/org.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/programs.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/provider_credentials.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/signatures.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/tool_servers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/traces.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/commands/valsets.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/credentials.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/deprecation.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/pagination.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/trace_render.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/cli/verbs.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/client.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/configuration.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/context.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/dataset_sync.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/datasets.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/decorators.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/deployment.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/execution.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/exporter.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/exporter_http.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/helpers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/identity.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/imaging.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/decode.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/encode.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/hash.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/program.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/reflection.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/types.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/ir/xxh64.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/mcp_servers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/optimization.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/optimizers/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/optimizers/gepa.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/optimizers/gepa_callback.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/optimizers/resume.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/optimizers/tracker.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/retry.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/cmpnd/traces.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/2026-04-02-code-review.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/cli.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/client.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/custom-proposer.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/plans/2026-04-05-train-val-dataset-capture-design.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/plans/2026-04-05-train-val-dataset-capture.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/docs/reading-traces.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/examples/generalizing_proposer.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/examples/local_ollama.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/LINEAGE.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/README.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/bodies.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/exporter_http.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/protocols.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/py.typed +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/request_log.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/cmpnd_ingest/retry.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/ingest/pyproject.toml +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/audit-sdist.sh +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/capture_review_run.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/check_release_artifacts.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/fixtures/review_run.json.gz +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/gepa_resume_bug.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/require_version_bump.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/scripts/seed_review_run.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/stubs/dspy/__init__.pyi +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/stubs/dspy/primitives/__init__.pyi +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/stubs/dspy/primitives/prediction.pyi +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/stubs/dspy/utils/__init__.pyi +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/stubs/dspy/utils/callback.pyi +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/_exporter_helpers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/_fake_clock.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/_identity_helpers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/conftest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/committee_task.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/conftest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/data/committee_train.ndjson.gz +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/data/committee_val.ndjson.gz +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/fake_lm.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/helpers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/target_capabilities.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_capture_prompts.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_composite_module.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_dataset_auto_creation.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_deploy_execute.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_deploy_programs.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_edge_cases.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_example_hash.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_gepa_optimization.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_interpreter_spans.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_lm_accuracy.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_module_tracing.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_optimization_lineage.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_optimization_metric_identity.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_optimize_failures.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_optimize_loop.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_optimize_server_owned_completion.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_parallel_export.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_project_attribution.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_react_tool_spans.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_save_load.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_trace_structure.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_traces_read.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/e2e/test_user_attribution.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/01_predict_qa.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/02_predict_renamed_field.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/03_chain_of_thought_qa.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/04_predict_richer_types.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/05_composite_two_predicts.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/06_multichaincomparison.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/07_program_of_thought.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/08_opaque_module.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/09_predict_pydantic_field.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/10_react.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/11_refine.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/12_retrieve.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/13_knn.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/14_parallel.json +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/identity_vectors/README.md +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/ingest_conformance.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/conftest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/hosted_optimize_probe.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/test_cli_deploy_smoketest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/test_logs_renders_traces.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/test_optimization_run.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/test_optimizing_progress.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/integration/test_unhappy_path.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/__main__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/drain.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/ingest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/latency_transport.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/rlm_shape.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/sweep.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/load/test_rlm_shape.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/conftest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/seed_parity.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_auth.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_datasets.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_deployments.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_evals.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_health.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_modules.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_optimizations.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_programs.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_signatures.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_trace_streaming.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/parity/test_traces.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/_provider.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/_reliability.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/_result.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/mint_bedrock_token.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_deploy_programs.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_multi_client_stress.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_optimize_client_committee.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_optimize_loop.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_optimize_loop_wasm.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_optimize_stress.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_provider_roles.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_real_provider_twins.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_reliability.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_reliability_classify.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_rlm_wasm.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/stress/test_trace_search.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_callback.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_candidate_lineage.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/conftest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_admin.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_api_client.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_credentials.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_login.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_main.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_org.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_pagination.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_provider_credentials.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_tool_servers.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_cli/test_top.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_client.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_clustering/__init__.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_clustering/conftest.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_clustering/test_edge_cases.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_clustering/test_hash_consistency.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_clustering/test_type_conversion.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_configuration.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_context.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_context_propagation.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_dataset_polling.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_dataset_sync.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_datasets.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_decorators.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_deploy.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_eval.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_eval_example_attribution.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_eval_start.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_eval_trace_grouping.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_exception_paths.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_execute.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_exporter_queue.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_exporter_real.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_fake_lm.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_gepa_callback.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_gepa_integration.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_gepa_resume.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_identity.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_imaging.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_import_weight.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_ingest_conformance.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_ingest_second_producer.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_integration.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_interpreter_callback.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_ir_golden_vectors.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_ir_roundtrip.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_models.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimization_start.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize_budget.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize_lift.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize_lms.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize_proposer.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_optimize_reflection_lm.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_packaging.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_packaging_callable_source.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_packaging_program_source.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_poll_until.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_program_lineage_persist.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_project_payload.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_repro_reactv2_bugs.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_require_version_bump.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_retry.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_rlm_patch.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_schemas.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_seed_review_run.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_trace_render.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_tracked_gepa.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_tracker_enrich.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/tests/test_truncation.py +0 -0
- {cmpnd-0.23.0 → cmpnd-0.23.2}/todo.md +0 -0
|
@@ -181,6 +181,27 @@ def _normalize_usage(usage: Any) -> dict[str, int | None]:
|
|
|
181
181
|
columns behind them are ``NOT NULL``). ``reasoning_tokens`` and
|
|
182
182
|
``cache_write_tokens`` come back as ``None`` when the provider said nothing.
|
|
183
183
|
"""
|
|
184
|
+
evidence = _normalize_usage_evidence(usage)
|
|
185
|
+
prompt = evidence["prompt_tokens"]
|
|
186
|
+
completion = evidence["completion_tokens"]
|
|
187
|
+
total = evidence["total_tokens"]
|
|
188
|
+
if total is None and (prompt is not None or completion is not None):
|
|
189
|
+
# The legacy span projection has always derived a total. The evidence
|
|
190
|
+
# object keeps an omitted total as null and preserves the raw block.
|
|
191
|
+
total = (prompt or 0) + (completion or 0)
|
|
192
|
+
|
|
193
|
+
return {
|
|
194
|
+
"prompt_tokens": prompt or 0,
|
|
195
|
+
"completion_tokens": completion or 0,
|
|
196
|
+
"total_tokens": total or 0,
|
|
197
|
+
"cached_prompt_tokens": evidence["cache_read_tokens"] or 0,
|
|
198
|
+
"reasoning_tokens": evidence["reasoning_tokens"],
|
|
199
|
+
"cache_write_tokens": evidence["cache_write_tokens"],
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _normalize_usage_evidence(usage: Any) -> dict[str, int | None]:
|
|
204
|
+
"""Normalize queryable counters without flattening absence into zero."""
|
|
184
205
|
prompt = _reported_count(
|
|
185
206
|
_usage_field(usage, "prompt_tokens"),
|
|
186
207
|
_usage_field(usage, "input_tokens"),
|
|
@@ -190,11 +211,6 @@ def _normalize_usage(usage: Any) -> dict[str, int | None]:
|
|
|
190
211
|
_usage_field(usage, "output_tokens"),
|
|
191
212
|
)
|
|
192
213
|
total = _reported_count(_usage_field(usage, "total_tokens"))
|
|
193
|
-
if total is None and (prompt is not None or completion is not None):
|
|
194
|
-
# Derived only where the provider omitted it — a reported total is
|
|
195
|
-
# evidence and stays as sent, even when the split disagrees with it.
|
|
196
|
-
total = (prompt or 0) + (completion or 0)
|
|
197
|
-
|
|
198
214
|
cached = _reported_count(
|
|
199
215
|
_usage_detail(usage, "prompt_tokens_details", "cached_tokens"),
|
|
200
216
|
_usage_detail(usage, "input_tokens_details", "cached_tokens"),
|
|
@@ -209,20 +225,159 @@ def _normalize_usage(usage: Any) -> dict[str, int | None]:
|
|
|
209
225
|
cache_write = _reported_count(
|
|
210
226
|
_usage_field(usage, "cache_write_tokens"),
|
|
211
227
|
_usage_field(usage, "cache_creation_input_tokens"),
|
|
228
|
+
_usage_detail(usage, "prompt_tokens_details", "cache_creation_tokens"),
|
|
229
|
+
)
|
|
230
|
+
cache_write_5m = _reported_count(
|
|
231
|
+
_usage_detail(usage, "cache_creation", "ephemeral_5m_input_tokens"),
|
|
232
|
+
_usage_detail(usage, "cache_creation_token_details", "ephemeral_5m_input_tokens"),
|
|
233
|
+
)
|
|
234
|
+
cache_write_1h = _reported_count(
|
|
235
|
+
_usage_detail(usage, "cache_creation", "ephemeral_1h_input_tokens"),
|
|
236
|
+
_usage_detail(usage, "cache_creation_token_details", "ephemeral_1h_input_tokens"),
|
|
237
|
+
)
|
|
238
|
+
audio_prompt = _reported_count(
|
|
239
|
+
_usage_detail(usage, "prompt_tokens_details", "audio_tokens"),
|
|
240
|
+
_usage_detail(usage, "input_tokens_details", "audio_tokens"),
|
|
241
|
+
_usage_field(usage, "input_audio_tokens"),
|
|
242
|
+
_usage_field(usage, "audio_prompt_tokens"),
|
|
243
|
+
)
|
|
244
|
+
audio_completion = _reported_count(
|
|
245
|
+
_usage_detail(usage, "completion_tokens_details", "audio_tokens"),
|
|
246
|
+
_usage_detail(usage, "output_tokens_details", "audio_tokens"),
|
|
247
|
+
_usage_field(usage, "output_audio_tokens"),
|
|
248
|
+
_usage_field(usage, "audio_completion_tokens"),
|
|
212
249
|
)
|
|
213
250
|
|
|
214
251
|
return {
|
|
215
|
-
"prompt_tokens": prompt
|
|
216
|
-
"completion_tokens": completion
|
|
217
|
-
"total_tokens": total
|
|
218
|
-
|
|
219
|
-
# inside the prompt total, so the subset invariant holds by construction.
|
|
220
|
-
"cached_prompt_tokens": cached or 0,
|
|
252
|
+
"prompt_tokens": prompt,
|
|
253
|
+
"completion_tokens": completion,
|
|
254
|
+
"total_tokens": total,
|
|
255
|
+
"cache_read_tokens": cached,
|
|
221
256
|
"reasoning_tokens": reasoning,
|
|
222
257
|
"cache_write_tokens": cache_write,
|
|
258
|
+
"cache_write_5m_tokens": cache_write_5m,
|
|
259
|
+
"cache_write_1h_tokens": cache_write_1h,
|
|
260
|
+
"audio_prompt_tokens": audio_prompt,
|
|
261
|
+
"audio_completion_tokens": audio_completion,
|
|
223
262
|
}
|
|
224
263
|
|
|
225
264
|
|
|
265
|
+
_MISSING = object()
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _value_field(value: Any, key: str) -> Any:
|
|
269
|
+
"""Read a field from a mapping, DSPy model, or response object."""
|
|
270
|
+
if value is None:
|
|
271
|
+
return None
|
|
272
|
+
if isinstance(value, dict):
|
|
273
|
+
return value.get(key)
|
|
274
|
+
getter = getattr(value, "get", None)
|
|
275
|
+
if callable(getter):
|
|
276
|
+
try:
|
|
277
|
+
found = getter(key, _MISSING)
|
|
278
|
+
except TypeError:
|
|
279
|
+
try:
|
|
280
|
+
found = getter(key)
|
|
281
|
+
except (KeyError, TypeError):
|
|
282
|
+
found = _MISSING
|
|
283
|
+
if found is not _MISSING:
|
|
284
|
+
return found
|
|
285
|
+
return getattr(value, key, None)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _text_field(value: Any, key: str) -> str | None:
|
|
289
|
+
found = _value_field(value, key)
|
|
290
|
+
if isinstance(found, str) and found:
|
|
291
|
+
return found
|
|
292
|
+
return None
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _bool_field(value: Any, key: str) -> bool | None:
|
|
296
|
+
found = _value_field(value, key)
|
|
297
|
+
return found if isinstance(found, bool) else None
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def _present_field(value: Any, key: str) -> tuple[bool, Any]:
|
|
301
|
+
"""Read a field while preserving whether the object declared it."""
|
|
302
|
+
if isinstance(value, dict):
|
|
303
|
+
return key in value, value.get(key)
|
|
304
|
+
model_fields = getattr(type(value), "model_fields", None)
|
|
305
|
+
if isinstance(model_fields, dict) and key in model_fields:
|
|
306
|
+
return True, getattr(value, key, None)
|
|
307
|
+
values = getattr(value, "__dict__", None)
|
|
308
|
+
if isinstance(values, dict) and key in values:
|
|
309
|
+
return True, values.get(key)
|
|
310
|
+
return False, None
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _history_usage(entry: Any) -> tuple[bool, Any]:
|
|
314
|
+
"""Return whether history carried usage and the value it carried."""
|
|
315
|
+
response = _value_field(entry, "response")
|
|
316
|
+
response_declares_usage, response_usage = _present_field(response, "usage")
|
|
317
|
+
if response_declares_usage:
|
|
318
|
+
return response_usage is not None, response_usage
|
|
319
|
+
if isinstance(entry, dict):
|
|
320
|
+
usage = entry.get("usage")
|
|
321
|
+
return "usage" in entry and usage is not None, usage
|
|
322
|
+
usage = _value_field(entry, "usage")
|
|
323
|
+
return usage is not None, usage
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _usage_dialect(instance: Any, usage: Any) -> str | None:
|
|
327
|
+
"""Name the response surface only when its shape establishes one."""
|
|
328
|
+
model_type = _text_field(instance, "model_type")
|
|
329
|
+
if model_type == "responses":
|
|
330
|
+
return "responses"
|
|
331
|
+
if model_type in {"chat", "text"}:
|
|
332
|
+
return "chat_completions"
|
|
333
|
+
if _usage_field(usage, "inputTokens") is not None or _usage_field(usage, "outputTokens") is not None:
|
|
334
|
+
return "converse"
|
|
335
|
+
if _usage_field(usage, "prompt_tokens") is not None or _usage_field(usage, "completion_tokens") is not None:
|
|
336
|
+
return "chat_completions"
|
|
337
|
+
return None
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _cache_ttl_requested(*values: Any) -> str | None:
|
|
341
|
+
"""Return one unambiguous requested cache TTL from request data."""
|
|
342
|
+
found: set[str] = set()
|
|
343
|
+
|
|
344
|
+
def visit(value: Any, in_cache_control: bool = False) -> None:
|
|
345
|
+
if isinstance(value, dict):
|
|
346
|
+
for key, child in value.items():
|
|
347
|
+
if key in {"cache_ttl", "cache_ttl_requested"} and child is not None:
|
|
348
|
+
found.add(str(child))
|
|
349
|
+
elif in_cache_control and key == "ttl" and child is not None:
|
|
350
|
+
found.add(str(child))
|
|
351
|
+
else:
|
|
352
|
+
visit(child, in_cache_control or key == "cache_control")
|
|
353
|
+
elif isinstance(value, (list, tuple)):
|
|
354
|
+
for child in value:
|
|
355
|
+
visit(child, in_cache_control)
|
|
356
|
+
|
|
357
|
+
for value in values:
|
|
358
|
+
visit(value)
|
|
359
|
+
return next(iter(found)) if len(found) == 1 else None
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _request_selector(entry: Any, instance: Any, key: str) -> str | None:
|
|
363
|
+
history_kwargs = _value_field(entry, "kwargs")
|
|
364
|
+
return _text_field(history_kwargs, key) or _text_field(getattr(instance, "kwargs", None), key)
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _response_text(response: Any, key: str) -> str | None:
|
|
368
|
+
"""Read provider response metadata from DSPy's supported containers."""
|
|
369
|
+
for container in (
|
|
370
|
+
response,
|
|
371
|
+
_value_field(response, "provider_data"),
|
|
372
|
+
_value_field(response, "metadata"),
|
|
373
|
+
_value_field(response, "_hidden_params"),
|
|
374
|
+
):
|
|
375
|
+
found = _text_field(container, key)
|
|
376
|
+
if found:
|
|
377
|
+
return found
|
|
378
|
+
return None
|
|
379
|
+
|
|
380
|
+
|
|
226
381
|
def _finish_reason(response: Any) -> str | None:
|
|
227
382
|
"""Read why the model stopped, from whichever response shape answered.
|
|
228
383
|
|
|
@@ -991,6 +1146,13 @@ class _LMMixin(_CallbackBase):
|
|
|
991
1146
|
lm_kwargs = {
|
|
992
1147
|
k: v for k, v in getattr(instance, "kwargs", {}).items() if k not in {"api_key", "api_base", "api_secret"}
|
|
993
1148
|
}
|
|
1149
|
+
span.lm_usage = models.LMUsageObservationV1(
|
|
1150
|
+
sdk_call_id=call_id,
|
|
1151
|
+
model_requested=span.model_name,
|
|
1152
|
+
service_tier_requested=_text_field(lm_kwargs, "service_tier"),
|
|
1153
|
+
cache_ttl_requested=_cache_ttl_requested(lm_kwargs, inputs),
|
|
1154
|
+
binding=_text_field(instance, "binding"),
|
|
1155
|
+
)
|
|
994
1156
|
span.attributes.update(
|
|
995
1157
|
{
|
|
996
1158
|
"lm.model_type": getattr(instance, "model_type", "unknown"),
|
|
@@ -1027,38 +1189,95 @@ class _LMMixin(_CallbackBase):
|
|
|
1027
1189
|
|
|
1028
1190
|
span.end_time = datetime.now(timezone.utc)
|
|
1029
1191
|
|
|
1192
|
+
latest_entry: Any = None
|
|
1193
|
+
response: Any = None
|
|
1194
|
+
usage: Any = None
|
|
1195
|
+
usage_reported = False
|
|
1196
|
+
if instance and hasattr(instance, "history") and instance.history:
|
|
1197
|
+
try:
|
|
1198
|
+
latest_entry = instance.history[-1]
|
|
1199
|
+
usage_reported, usage = _history_usage(latest_entry)
|
|
1200
|
+
response = _value_field(latest_entry, "response")
|
|
1201
|
+
logger.debug(
|
|
1202
|
+
f"LM history usage for {span.model_name}: "
|
|
1203
|
+
f"history_len={len(instance.history)}, "
|
|
1204
|
+
f"usage_type={type(usage).__name__}, "
|
|
1205
|
+
f"usage={usage}"
|
|
1206
|
+
)
|
|
1207
|
+
except (IndexError, AttributeError, TypeError) as e:
|
|
1208
|
+
logger.debug(f"LM history access failed: {e}")
|
|
1209
|
+
else:
|
|
1210
|
+
has_hist = hasattr(instance, "history") if instance else False
|
|
1211
|
+
hist_len = len(instance.history) if has_hist else "N/A"
|
|
1212
|
+
logger.debug(
|
|
1213
|
+
f"LM history unavailable: instance={instance is not None}, "
|
|
1214
|
+
f"has_history={has_hist}, history_len={hist_len}"
|
|
1215
|
+
)
|
|
1216
|
+
|
|
1217
|
+
if not usage_reported and isinstance(outputs, dict) and "usage" in outputs:
|
|
1218
|
+
usage = outputs.get("usage")
|
|
1219
|
+
usage_reported = usage is not None
|
|
1220
|
+
if response is None:
|
|
1221
|
+
response = outputs
|
|
1222
|
+
|
|
1223
|
+
observation = span.lm_usage
|
|
1224
|
+
if observation is not None:
|
|
1225
|
+
observation.observed_at = span.end_time
|
|
1226
|
+
observation.outcome = "success" if exception is None else None
|
|
1227
|
+
cache_hit = _bool_field(response, "cache_hit")
|
|
1228
|
+
if cache_hit is None:
|
|
1229
|
+
cache_hit = _bool_field(_value_field(response, "_hidden_params"), "cache_hit")
|
|
1230
|
+
observation.dspy_client_cache_hit = cache_hit
|
|
1231
|
+
observation.usage_reported = usage_reported and cache_hit is not True
|
|
1232
|
+
serialized_usage = models._safe_serialize(usage)
|
|
1233
|
+
observation.usage_raw = (
|
|
1234
|
+
serialized_usage if observation.usage_reported and isinstance(serialized_usage, dict) else None
|
|
1235
|
+
)
|
|
1236
|
+
observation.usage_dialect = _usage_dialect(instance, usage)
|
|
1237
|
+
if observation.usage_dialect in {"chat_completions", "responses"}:
|
|
1238
|
+
observation.prompt_inclusion = "includes_cache_reads"
|
|
1239
|
+
elif observation.usage_dialect == "converse":
|
|
1240
|
+
observation.prompt_inclusion = "excludes_cache_reads"
|
|
1241
|
+
|
|
1242
|
+
if usage_reported and usage is not None:
|
|
1243
|
+
evidence = _normalize_usage_evidence(usage)
|
|
1244
|
+
for name, value in evidence.items():
|
|
1245
|
+
setattr(observation, name, value)
|
|
1246
|
+
if (
|
|
1247
|
+
observation.prompt_tokens is not None
|
|
1248
|
+
and observation.cache_read_tokens is not None
|
|
1249
|
+
and observation.prompt_inclusion == "includes_cache_reads"
|
|
1250
|
+
and observation.cache_read_tokens <= observation.prompt_tokens
|
|
1251
|
+
):
|
|
1252
|
+
observation.fresh_input_tokens = observation.prompt_tokens - observation.cache_read_tokens
|
|
1253
|
+
elif observation.prompt_tokens is not None and observation.prompt_inclusion == "excludes_cache_reads":
|
|
1254
|
+
observation.fresh_input_tokens = observation.prompt_tokens
|
|
1255
|
+
|
|
1256
|
+
requested_model = _text_field(latest_entry, "model")
|
|
1257
|
+
if requested_model:
|
|
1258
|
+
observation.model_requested = requested_model
|
|
1259
|
+
reported_model = _text_field(latest_entry, "response_model") or _text_field(response, "model")
|
|
1260
|
+
if reported_model:
|
|
1261
|
+
observation.model_reported = reported_model
|
|
1262
|
+
requested_tier = _request_selector(latest_entry, instance, "service_tier")
|
|
1263
|
+
if requested_tier:
|
|
1264
|
+
observation.service_tier_requested = requested_tier
|
|
1265
|
+
requested_ttl = _cache_ttl_requested(_value_field(latest_entry, "kwargs"))
|
|
1266
|
+
if requested_ttl:
|
|
1267
|
+
observation.cache_ttl_requested = requested_ttl
|
|
1268
|
+
effective_tier = _response_text(response, "service_tier")
|
|
1269
|
+
if effective_tier:
|
|
1270
|
+
observation.service_tier_effective = effective_tier
|
|
1271
|
+
observation.provider_response_id = _response_text(response, "response_id") or _response_text(response, "id")
|
|
1272
|
+
observation.egress_call_id = _response_text(response, "egress_call_id") or _text_field(
|
|
1273
|
+
latest_entry, "egress_call_id"
|
|
1274
|
+
)
|
|
1275
|
+
|
|
1030
1276
|
if exception:
|
|
1031
1277
|
span.set_error(exception)
|
|
1032
1278
|
else:
|
|
1033
1279
|
span.status = models.TraceStatus.OK
|
|
1034
1280
|
|
|
1035
|
-
# Extract token usage - try LM's history first (where DSPy stores it),
|
|
1036
|
-
# then fall back to outputs dict (for backwards compatibility / testing)
|
|
1037
|
-
usage = {}
|
|
1038
|
-
if instance and hasattr(instance, "history") and instance.history:
|
|
1039
|
-
try:
|
|
1040
|
-
latest_entry = instance.history[-1]
|
|
1041
|
-
usage = latest_entry.get("usage", {})
|
|
1042
|
-
logger.debug(
|
|
1043
|
-
f"LM history usage for {span.model_name}: "
|
|
1044
|
-
f"history_len={len(instance.history)}, "
|
|
1045
|
-
f"usage_type={type(usage).__name__}, "
|
|
1046
|
-
f"usage={usage}"
|
|
1047
|
-
)
|
|
1048
|
-
except (IndexError, AttributeError, TypeError) as e:
|
|
1049
|
-
logger.debug(f"LM history access failed: {e}")
|
|
1050
|
-
else:
|
|
1051
|
-
has_hist = hasattr(instance, "history") if instance else False
|
|
1052
|
-
hist_len = len(instance.history) if has_hist else "N/A"
|
|
1053
|
-
logger.debug(
|
|
1054
|
-
f"LM history unavailable: instance={instance is not None}, "
|
|
1055
|
-
f"has_history={has_hist}, history_len={hist_len}"
|
|
1056
|
-
)
|
|
1057
|
-
|
|
1058
|
-
# Fallback: check outputs dict for usage (legacy / test compatibility)
|
|
1059
|
-
if not usage and isinstance(outputs, dict):
|
|
1060
|
-
usage = outputs.get("usage", {})
|
|
1061
|
-
|
|
1062
1281
|
if usage:
|
|
1063
1282
|
counts = _normalize_usage(usage)
|
|
1064
1283
|
span.prompt_tokens = counts["prompt_tokens"] or 0
|
|
@@ -1072,14 +1291,9 @@ class _LMMixin(_CallbackBase):
|
|
|
1072
1291
|
logger.debug(f"NO usage data for {span.model_name} (call_id={call_id})")
|
|
1073
1292
|
|
|
1074
1293
|
# Extract finish_reason from LM history response
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
finish_reason = _finish_reason(latest_entry.get("response"))
|
|
1079
|
-
if finish_reason:
|
|
1080
|
-
span.set_attribute("lm.finish_reason", finish_reason)
|
|
1081
|
-
except (IndexError, AttributeError, TypeError):
|
|
1082
|
-
pass
|
|
1294
|
+
finish_reason = _finish_reason(response)
|
|
1295
|
+
if finish_reason:
|
|
1296
|
+
span.set_attribute("lm.finish_reason", finish_reason)
|
|
1083
1297
|
|
|
1084
1298
|
# Capture response outputs (truncated)
|
|
1085
1299
|
if outputs and self._capture_outputs:
|
|
@@ -192,11 +192,11 @@ def _exchange_code(endpoint: str, code: str) -> str:
|
|
|
192
192
|
bearer); the server consumes the code on read so it cannot be replayed.
|
|
193
193
|
"""
|
|
194
194
|
# Bare HTTP POST — ApiClient requires an api_key we don't have yet.
|
|
195
|
-
import json
|
|
195
|
+
import json
|
|
196
196
|
import urllib.request
|
|
197
197
|
|
|
198
198
|
url = f"{endpoint.rstrip('/')}/api/v1/auth/cli/exchange"
|
|
199
|
-
body =
|
|
199
|
+
body = json.dumps({"code": code}).encode("utf-8")
|
|
200
200
|
req = urllib.request.Request(
|
|
201
201
|
url,
|
|
202
202
|
data=body,
|
|
@@ -205,7 +205,7 @@ def _exchange_code(endpoint: str, code: str) -> str:
|
|
|
205
205
|
)
|
|
206
206
|
try:
|
|
207
207
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
208
|
-
payload =
|
|
208
|
+
payload = json.loads(resp.read().decode("utf-8"))
|
|
209
209
|
except Exception as exc:
|
|
210
210
|
raise api_client.CliError(f"Failed to exchange login code: {exc}") from exc
|
|
211
211
|
token = payload.get("api_key") if isinstance(payload, dict) else None
|
|
@@ -985,11 +985,11 @@ def cmd_run(client: api_client.ApiClient, args: Any) -> None:
|
|
|
985
985
|
|
|
986
986
|
|
|
987
987
|
def register(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
988
|
-
import argparse
|
|
988
|
+
import argparse
|
|
989
989
|
|
|
990
990
|
from .. import verbs
|
|
991
991
|
|
|
992
|
-
raw =
|
|
992
|
+
raw = argparse.RawDescriptionHelpFormatter
|
|
993
993
|
|
|
994
994
|
# Resolve handler refs through VERBS so adding a CLI subparser without
|
|
995
995
|
# a corresponding entry in cmpnd.cli.verbs (or vice versa) trips
|
|
@@ -25,7 +25,7 @@ The registry handles caching, ``__dir__``, ``__iter__``, indexing,
|
|
|
25
25
|
|
|
26
26
|
from __future__ import annotations
|
|
27
27
|
|
|
28
|
-
import json
|
|
28
|
+
import json
|
|
29
29
|
import os
|
|
30
30
|
import sys
|
|
31
31
|
from pathlib import Path
|
|
@@ -418,7 +418,7 @@ class Deployment(Entity):
|
|
|
418
418
|
raise TypeError("pass either a body dict or kwargs, not both")
|
|
419
419
|
if body is None:
|
|
420
420
|
body = kwargs
|
|
421
|
-
body_str = body if isinstance(body, str) else
|
|
421
|
+
body_str = body if isinstance(body, str) else json.dumps(body)
|
|
422
422
|
commands.top.cmd_run(self._client, _args(id=self.id, body=body_str))
|
|
423
423
|
|
|
424
424
|
def watch(self, timeout: int = 600) -> None:
|
|
@@ -279,7 +279,7 @@ class EvalHandler:
|
|
|
279
279
|
# not a position — is what a result is matched on.
|
|
280
280
|
traces_by_id = {str(t.trace_id): t for t in eval_traces}
|
|
281
281
|
# Imported here, not at module scope: callback imports eval_handler.
|
|
282
|
-
from . import callback
|
|
282
|
+
from . import callback
|
|
283
283
|
|
|
284
284
|
logger.debug(f"Collected {len(eval_traces)} traces for eval run")
|
|
285
285
|
for i, t in enumerate(eval_traces[:3]): # Show first 3
|
|
@@ -341,7 +341,7 @@ class EvalHandler:
|
|
|
341
341
|
# positional fallback: a result carrying no stamp is left
|
|
342
342
|
# unlinked, because a wrong trace, a wrong token count and a
|
|
343
343
|
# wrong cost are worse than none.
|
|
344
|
-
trace = traces_by_id.get(
|
|
344
|
+
trace = traces_by_id.get(callback.trace_id(prediction) or "")
|
|
345
345
|
if trace is not None:
|
|
346
346
|
example_builder.trace_id = trace.trace_id
|
|
347
347
|
|
|
@@ -2,13 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
|
+
import hashlib
|
|
5
6
|
import json
|
|
6
7
|
import re
|
|
7
8
|
from dataclasses import dataclass, field
|
|
8
9
|
from datetime import datetime, timezone
|
|
9
10
|
from enum import Enum
|
|
10
11
|
from typing import Any
|
|
11
|
-
from uuid import UUID
|
|
12
|
+
from uuid import UUID, uuid4
|
|
12
13
|
|
|
13
14
|
from . import identity, imaging
|
|
14
15
|
|
|
@@ -412,6 +413,91 @@ class TraceStatus(str, Enum):
|
|
|
412
413
|
ERROR = "error"
|
|
413
414
|
|
|
414
415
|
|
|
416
|
+
@dataclass
|
|
417
|
+
class LMUsageObservationV1:
|
|
418
|
+
"""Provider-usage evidence attached to one SDK-observed LM call.
|
|
419
|
+
|
|
420
|
+
Optional counters preserve the distinction between an absent measurement
|
|
421
|
+
and a reported zero. ``provider_cost`` accepts only a provider-origin amount.
|
|
422
|
+
DSPy and LiteLLM calculated costs do not belong here.
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
source_observation_id: str = field(default_factory=lambda: str(uuid4()))
|
|
426
|
+
observed_at: datetime | None = None
|
|
427
|
+
usage_reported: bool = False
|
|
428
|
+
usage_raw: dict[str, Any] | None = None
|
|
429
|
+
usage_dialect: str | None = None
|
|
430
|
+
normalization_version: int = 1
|
|
431
|
+
prompt_inclusion: str | None = None
|
|
432
|
+
model_requested: str | None = None
|
|
433
|
+
model_reported: str | None = None
|
|
434
|
+
service_tier_requested: str | None = None
|
|
435
|
+
service_tier_effective: str | None = None
|
|
436
|
+
cache_ttl_requested: str | None = None
|
|
437
|
+
provider_cost: str | None = None
|
|
438
|
+
provider_cost_currency: str | None = None
|
|
439
|
+
dspy_client_cache_hit: bool | None = None
|
|
440
|
+
binding: str | None = None
|
|
441
|
+
sdk_call_id: str | None = None
|
|
442
|
+
provider_response_id: str | None = None
|
|
443
|
+
egress_call_id: str | None = None
|
|
444
|
+
prompt_tokens: int | None = None
|
|
445
|
+
completion_tokens: int | None = None
|
|
446
|
+
total_tokens: int | None = None
|
|
447
|
+
fresh_input_tokens: int | None = None
|
|
448
|
+
cache_read_tokens: int | None = None
|
|
449
|
+
cache_write_tokens: int | None = None
|
|
450
|
+
cache_write_5m_tokens: int | None = None
|
|
451
|
+
cache_write_1h_tokens: int | None = None
|
|
452
|
+
reasoning_tokens: int | None = None
|
|
453
|
+
audio_prompt_tokens: int | None = None
|
|
454
|
+
audio_completion_tokens: int | None = None
|
|
455
|
+
outcome: str | None = None
|
|
456
|
+
|
|
457
|
+
def to_api_dict(self) -> dict[str, Any]:
|
|
458
|
+
"""Return the versioned wire object with a deterministic digest."""
|
|
459
|
+
payload = {
|
|
460
|
+
"schema_version": 1,
|
|
461
|
+
"source_observation_id": self.source_observation_id,
|
|
462
|
+
"observed_at": self.observed_at.isoformat() if self.observed_at else None,
|
|
463
|
+
"usage_reported": self.usage_reported,
|
|
464
|
+
"usage_raw": _safe_serialize(self.usage_raw),
|
|
465
|
+
"usage_dialect": self.usage_dialect,
|
|
466
|
+
"normalization_version": self.normalization_version,
|
|
467
|
+
"prompt_inclusion": self.prompt_inclusion,
|
|
468
|
+
"model_requested": self.model_requested,
|
|
469
|
+
"model_reported": self.model_reported,
|
|
470
|
+
"service_tier_requested": self.service_tier_requested,
|
|
471
|
+
"service_tier_effective": self.service_tier_effective,
|
|
472
|
+
"cache_ttl_requested": self.cache_ttl_requested,
|
|
473
|
+
"provider_cost": self.provider_cost,
|
|
474
|
+
"provider_cost_currency": self.provider_cost_currency,
|
|
475
|
+
"dspy_client_cache_hit": self.dspy_client_cache_hit,
|
|
476
|
+
"binding": self.binding,
|
|
477
|
+
"sdk_call_id": self.sdk_call_id,
|
|
478
|
+
"provider_response_id": self.provider_response_id,
|
|
479
|
+
"egress_call_id": self.egress_call_id,
|
|
480
|
+
"prompt_tokens": self.prompt_tokens,
|
|
481
|
+
"completion_tokens": self.completion_tokens,
|
|
482
|
+
"total_tokens": self.total_tokens,
|
|
483
|
+
"fresh_input_tokens": self.fresh_input_tokens,
|
|
484
|
+
"cache_read_tokens": self.cache_read_tokens,
|
|
485
|
+
"cache_write_tokens": self.cache_write_tokens,
|
|
486
|
+
"cache_write_5m_tokens": self.cache_write_5m_tokens,
|
|
487
|
+
"cache_write_1h_tokens": self.cache_write_1h_tokens,
|
|
488
|
+
"reasoning_tokens": self.reasoning_tokens,
|
|
489
|
+
"audio_prompt_tokens": self.audio_prompt_tokens,
|
|
490
|
+
"audio_completion_tokens": self.audio_completion_tokens,
|
|
491
|
+
"outcome": self.outcome,
|
|
492
|
+
}
|
|
493
|
+
safe_payload = _safe_serialize(payload)
|
|
494
|
+
if not isinstance(safe_payload, dict):
|
|
495
|
+
raise TypeError("LM usage observation did not serialize to a JSON object")
|
|
496
|
+
canonical = json.dumps(safe_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
497
|
+
safe_payload["source_payload_digest"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
498
|
+
return safe_payload
|
|
499
|
+
|
|
500
|
+
|
|
415
501
|
@dataclass
|
|
416
502
|
class SpanBuilder:
|
|
417
503
|
"""Mutable span builder for collecting data during execution."""
|
|
@@ -469,6 +555,7 @@ class SpanBuilder:
|
|
|
469
555
|
# Normalized by callback._normalize_usage.
|
|
470
556
|
reasoning_tokens: int | None = None
|
|
471
557
|
cache_write_tokens: int | None = None
|
|
558
|
+
lm_usage: LMUsageObservationV1 | None = None
|
|
472
559
|
# Left unset by the SDK. A dollar figure is a derivation from these counts
|
|
473
560
|
# and a dated rate catalog, and only serve holds the catalog; the SDK's job
|
|
474
561
|
# is to report what the provider said. serve still reads the wire field, so
|
|
@@ -582,6 +669,7 @@ class SpanBuilder:
|
|
|
582
669
|
"cached_prompt_tokens": self.cached_prompt_tokens,
|
|
583
670
|
"reasoning_tokens": self.reasoning_tokens,
|
|
584
671
|
"cache_write_tokens": self.cache_write_tokens,
|
|
672
|
+
"lm_usage": self.lm_usage.to_api_dict() if self.lm_usage else None,
|
|
585
673
|
"cost": self.cost,
|
|
586
674
|
"demos": _safe_serialize(self.demos),
|
|
587
675
|
"demo_count": self.demo_count if self.demo_count else len(self.demos),
|
|
@@ -544,9 +544,9 @@ def _resolve_an_mcp_tool(
|
|
|
544
544
|
which no resolution can rescue — and they can fix it in one line, so the message
|
|
545
545
|
shows the line.
|
|
546
546
|
"""
|
|
547
|
-
from . import mcp_servers
|
|
547
|
+
from . import mcp_servers
|
|
548
548
|
|
|
549
|
-
reference =
|
|
549
|
+
reference = mcp_servers.mcp_tool_reference(tool)
|
|
550
550
|
if reference is not None:
|
|
551
551
|
entry = pinned.get((reference.server_url, reference.tool_name))
|
|
552
552
|
if entry is None:
|
|
@@ -560,7 +560,7 @@ def _resolve_an_mcp_tool(
|
|
|
560
560
|
)
|
|
561
561
|
return entry
|
|
562
562
|
|
|
563
|
-
if
|
|
563
|
+
if mcp_servers.is_bare_mcp_tool(tool):
|
|
564
564
|
raise ValueError(
|
|
565
565
|
f"tool {name!r} was built with dspy.Tool.from_mcp_tool, which closes over a live MCP session: "
|
|
566
566
|
"the server it came from and the credential it was reached with are not recoverable from it, "
|
|
@@ -787,7 +787,7 @@ def generate_program_source(
|
|
|
787
787
|
pinned_mcp_tools = mcp_tools or {}
|
|
788
788
|
tools_dict = module_tools(module)
|
|
789
789
|
if tools_dict:
|
|
790
|
-
from . import mcp_servers
|
|
790
|
+
from . import mcp_servers
|
|
791
791
|
|
|
792
792
|
for name, tool in tools_dict.items():
|
|
793
793
|
if name == "finish":
|
|
@@ -798,7 +798,7 @@ def generate_program_source(
|
|
|
798
798
|
# instead of a body — see [_generate_mcp_tool_stub]. The `def`'s name is
|
|
799
799
|
# the MCP tool's own, because the declaration names the tool and the
|
|
800
800
|
# worker maps that name to the token it holds.
|
|
801
|
-
signature =
|
|
801
|
+
signature = mcp_servers.synthesized_signature(pinned)
|
|
802
802
|
lines.append(_generate_mcp_tool_stub(pinned, signature))
|
|
803
803
|
mcp_declarations.append(
|
|
804
804
|
{
|