cmpnd 0.44.0__tar.gz → 0.45.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cmpnd-0.44.0 → cmpnd-0.45.0}/PKG-INFO +1 -1
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/__init__.py +2 -1
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/callback.py +6 -1
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/lm_configuration.py +68 -4
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/optimization.py +17 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/sdk-instrumentation.md +21 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/pyproject.toml +1 -1
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/hosted_optimize_probe.py +58 -0
- cmpnd-0.45.0/tests/test_decision_client_spans.py +144 -0
- cmpnd-0.45.0/tests/test_system_one.py +106 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/uv.lock +1 -1
- {cmpnd-0.44.0 → cmpnd-0.45.0}/.gitignore +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/AGENTS.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/CLAUDE.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/README.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_assessments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_authored_import.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_deployments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_exporter_pool.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_gepa_patch.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_http.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_program_patch.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_rlm_patch.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_runtime_tools.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_timespec.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_traces.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/_valsets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/authored.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/api_client.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/admin.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/assessments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/auth.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/datasets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/deployments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/evals.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/login.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/optimizations.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/org.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/provider_credentials.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/signatures.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/tool_servers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/top.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/traces.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/commands/valsets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/credentials.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/deprecation.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/pagination.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/shell.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/trace_render.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/tree_help.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/cli/verbs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/client.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/configuration.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/context.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/dataset_sync.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/datasets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/decorators.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/deployment.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/deployment_metadata.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/deployment_runs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/eval_handler.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/execution.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/exporter.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/exporter_http.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/helpers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/identity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/imaging.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/decode.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/encode.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/hash.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/program.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/reflection.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/types.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/ir/xxh64.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/mcp_servers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/models.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/named_programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/optimizers/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/optimizers/gepa.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/optimizers/gepa_callback.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/optimizers/resume.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/optimizers/tracker.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/packaging.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/references.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/retry.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/cmpnd/traces.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/2026-04-02-code-review.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/cli.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/client.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/custom-proposer.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/plans/2026-04-05-train-val-dataset-capture-design.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/plans/2026-04-05-train-val-dataset-capture.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/docs/reading-traces.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/examples/generalizing_proposer.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/examples/local_ollama.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/LINEAGE.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/README.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/bodies.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/exporter_http.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/program_names.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/protocols.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/py.typed +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/request_log.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/retry.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/cmpnd_ingest/trace_facts.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/ingest/pyproject.toml +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/audit-sdist.sh +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/capture_review_run.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/check-pip-install.sh +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/check_release_artifacts.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/fixtures/review_run.json.gz +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/gepa_resume_bug.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/release-stress.sh +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/require_version_bump.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/seed_review_run.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/scripts/test-in-linux.sh +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/stubs/dspy/__init__.pyi +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/stubs/dspy/primitives/__init__.pyi +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/stubs/dspy/primitives/prediction.pyi +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/stubs/dspy/utils/__init__.pyi +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/stubs/dspy/utils/callback.pyi +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/stubs/dspy/utils/dummies.pyi +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/_deployment_signature_helpers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/_exporter_helpers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/_fake_clock.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/_identity_helpers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/_resolve_refs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/_traced_run.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/conftest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/committee_task.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/conftest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/data/committee_train.ndjson.gz +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/data/committee_val.ndjson.gz +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/fake_lm.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/helpers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/target_capabilities.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_assessments_cli.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_capture_prompts.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_composite_module.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_dataset_auto_creation.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_deploy_execute.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_deploy_pandas.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_deploy_programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_edge_cases.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_example_hash.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_gepa_optimization.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_interpreter_spans.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_lm_accuracy.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_module_tracing.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_optimization_lineage.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_optimization_metric_identity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_optimize_failures.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_optimize_loop.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_optimize_review_committee.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_optimize_server_owned_completion.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_parallel_export.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_project_attribution.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_react_tool_spans.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_save_load.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_trace_structure.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_traces_read.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_user_attribution.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/e2e/test_valsets_cli.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/01_predict_qa.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/02_predict_renamed_field.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/03_chain_of_thought_qa.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/04_predict_richer_types.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/05_composite_two_predicts.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/06_multichaincomparison.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/07_program_of_thought.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/08_opaque_module.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/09_predict_pydantic_field.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/10_react.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/11_refine.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/12_retrieve.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/13_knn.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/14_parallel.json +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/identity_vectors/README.md +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/ingest_conformance.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/authored_optimize/judge.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/authored_optimize/optimize_hosted.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/conftest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/hosted_mcp_tool_optimize_probe.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/optimization_run_probe.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/test_cli_deploy_smoketest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/test_logs_renders_traces.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/test_optimization_run.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/test_optimizing_progress.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/integration/test_unhappy_path.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/__main__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/drain.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/ingest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/latency_transport.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/rlm_shape.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/rlm_size_report.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/sweep.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/test_rlm_shape.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/load/test_rlm_size_report.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/conftest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/seed_parity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_auth.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_datasets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_deployments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_evals.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_health.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_modules.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_optimizations.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_signatures.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_trace_streaming.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/parity/test_traces.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/span_reference_fixture.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/_provider.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/_readiness.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/_reliability.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/_result.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/mint_bedrock_token.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_deploy_programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_multi_client_stress.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_optimize_client_committee.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_optimize_loop.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_optimize_loop_wasm.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_optimize_stress.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_provider_roles.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_readiness.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_real_provider_twins.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_reliability.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_reliability_classify.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_rlm_wasm.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/stress/test_trace_search.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_adapter_reentrancy.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_assessments_api.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_authored.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_authored_bundle_collisions.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_authored_metric_contract.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_authored_named_models.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_authored_optimize.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_callback.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_candidate_lineage.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/conftest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_admin.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_api_client.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_assessments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_auth.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_credentials.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_datasets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_deployment_runs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_deployments.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_evals.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_login.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_main.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_optimizations.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_org.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_pagination.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_parity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_programs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_provider_credentials.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_schema_sample.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_shell.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_signatures.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_tool_servers.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_top.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_traces.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_traces_search_wire.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_tree_help.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cli/test_valsets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_client.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_client_program_names.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_clustering/__init__.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_clustering/conftest.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_clustering/test_edge_cases.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_clustering/test_hash_consistency.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_clustering/test_type_conversion.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_cmp551_grouping_identity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_configuration.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_context.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_context_propagation.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_dataset_polling.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_dataset_sync.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_datasets.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_decorators.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_deploy.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_deploy_mcp_tool_references.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_deploy_program_id.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_deployment_metadata.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_deployment_runs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_deployments_read.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_eval.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_eval_example_attribution.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_eval_start.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_eval_trace_grouping.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_exception_paths.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_execute.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_execute_run_identity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_exporter.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_exporter_program_names.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_exporter_queue.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_exporter_real.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_fake_lm.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_gepa_callback.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_gepa_integration.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_gepa_output_normalization.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_gepa_resume.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_identity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_imaging.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_import_weight.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_ingest_conformance.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_ingest_second_producer.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_integration.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_interpreter_callback.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_ir_encoder_properties.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_ir_golden_vectors.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_ir_roundtrip.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_lm15_native_calls.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_lm_configuration.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_lm_usage_shapes.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_mcp_tools.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_models.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_multi_client_stress_readiness.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimization_progress.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimization_start.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize_budget.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize_lift.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize_lms.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize_proposer.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize_reflection_lm.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_optimize_terminal_failure.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_packaging.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_packaging_callable_source.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_packaging_mcp_tools.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_packaging_program_source.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_poll_until.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_program_identity.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_program_lineage_persist.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_programs_create.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_project_payload.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_real_provider_twins_readiness.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_reflector_properties.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_reflector_smoke.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_release_stress_script.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_repro_reactv2_bugs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_require_version_bump.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_retry.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_reversible_failures.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_rlm_patch.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_run_attribution_paths.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_runtime_tools.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_schemas.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_seed_review_run.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_signature_inputs.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_span_references.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_span_sent_once.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_trace_context_concurrency.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_trace_fact_wire_contract.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_trace_facts.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_trace_render.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_traces_read.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_tracked_gepa.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_tracker_enrich.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_truncation.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_valsets_api.py +0 -0
- {cmpnd-0.44.0 → cmpnd-0.45.0}/tests/test_xxh64.py +0 -0
|
@@ -46,7 +46,7 @@ from .deployment_runs import (
|
|
|
46
46
|
)
|
|
47
47
|
from .execution import DeployedProgram, RetryableProviderError, deployment_run_id, execute
|
|
48
48
|
from .exporter import flush_exporter, shutdown_exporter
|
|
49
|
-
from .lm_configuration import configure_lm
|
|
49
|
+
from .lm_configuration import SystemOne, configure_lm
|
|
50
50
|
from .mcp_servers import (
|
|
51
51
|
Bearer,
|
|
52
52
|
McpToolReference,
|
|
@@ -80,6 +80,7 @@ __all__ = [
|
|
|
80
80
|
# Configuration
|
|
81
81
|
"configure",
|
|
82
82
|
"configure_lm",
|
|
83
|
+
"SystemOne",
|
|
83
84
|
"get_config",
|
|
84
85
|
"require_config",
|
|
85
86
|
# Main callback
|
|
@@ -383,6 +383,10 @@ def _lm15_cache_hit(entry: Any, response: Any) -> bool | None:
|
|
|
383
383
|
|
|
384
384
|
def _usage_dialect(instance: Any, usage: Any) -> str | None:
|
|
385
385
|
"""Name the response surface only when its shape establishes one."""
|
|
386
|
+
# A decision client (DSPy's `TypeSafe`) speaks System One. Its usage names chat's token
|
|
387
|
+
# fields, so the shape alone would misname it. DSPy dispatches on the same flag.
|
|
388
|
+
if getattr(instance, "supports_decision_requests", False) is True:
|
|
389
|
+
return "system_one"
|
|
386
390
|
model_type = _text_field(instance, "model_type")
|
|
387
391
|
if model_type == "responses":
|
|
388
392
|
return "responses"
|
|
@@ -1489,7 +1493,8 @@ class _LMMixin(_CallbackBase):
|
|
|
1489
1493
|
observation.usage_dialect = _usage_dialect(instance, usage)
|
|
1490
1494
|
if observation.usage_dialect in {"chat_completions", "responses"}:
|
|
1491
1495
|
observation.prompt_inclusion = "includes_cache_reads"
|
|
1492
|
-
elif observation.usage_dialect
|
|
1496
|
+
elif observation.usage_dialect in {"converse", "system_one"}:
|
|
1497
|
+
# System One serves nothing from a cache, so its prompt is all fresh input.
|
|
1493
1498
|
observation.prompt_inclusion = "excludes_cache_reads"
|
|
1494
1499
|
|
|
1495
1500
|
if usage_reported and usage is not None:
|
|
@@ -9,11 +9,25 @@ class LMConfigurationError(ValueError):
|
|
|
9
9
|
"""A hosted binding declaration cannot be resolved consistently."""
|
|
10
10
|
|
|
11
11
|
|
|
12
|
-
class LMCapabilities(TypedDict):
|
|
13
|
-
"""Explicit adapter hints for a hosted model, independent of its provider.
|
|
12
|
+
class LMCapabilities(TypedDict, total=False):
|
|
13
|
+
"""Explicit adapter hints for a hosted model, independent of its provider.
|
|
14
|
+
|
|
15
|
+
``dialect`` is present only for a System One binding, which :class:`SystemOne`
|
|
16
|
+
writes. Absent, the binding is a chat model.
|
|
17
|
+
"""
|
|
14
18
|
|
|
15
19
|
supported_params: list[str]
|
|
16
20
|
supports_response_schema: bool
|
|
21
|
+
dialect: str
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
# The wire protocol a binding's model speaks, named for the protocol and never for a
|
|
25
|
+
# provider or a client library: a chat model answers chat completions, and a System One
|
|
26
|
+
# model (TypeSafe's System One protocol, which other providers serve too) answers a set
|
|
27
|
+
# of questions about a state.
|
|
28
|
+
CHAT_COMPLETIONS = "chat_completions"
|
|
29
|
+
SYSTEM_ONE = "system_one"
|
|
30
|
+
_DIALECTS = frozenset({CHAT_COMPLETIONS, SYSTEM_ONE})
|
|
17
31
|
|
|
18
32
|
|
|
19
33
|
def _validate_lm_capabilities(value: object) -> dict[str, LMCapabilities]:
|
|
@@ -27,7 +41,7 @@ def _validate_lm_capabilities(value: object) -> dict[str, LMCapabilities]:
|
|
|
27
41
|
)
|
|
28
42
|
if not isinstance(declaration, dict):
|
|
29
43
|
raise LMConfigurationError(f"lm_capabilities[{binding!r}] must be a capability declaration")
|
|
30
|
-
unknown = declaration.keys() - {"supported_params", "supports_response_schema"}
|
|
44
|
+
unknown = declaration.keys() - {"supported_params", "supports_response_schema", "dialect"}
|
|
31
45
|
if unknown:
|
|
32
46
|
raise LMConfigurationError(f"lm_capabilities[{binding!r}] has unknown fields: {sorted(unknown, key=str)}")
|
|
33
47
|
params = declaration.get("supported_params", [])
|
|
@@ -38,11 +52,24 @@ def _validate_lm_capabilities(value: object) -> dict[str, LMCapabilities]:
|
|
|
38
52
|
schema = declaration.get("supports_response_schema", False)
|
|
39
53
|
if not isinstance(schema, bool):
|
|
40
54
|
raise LMConfigurationError(f"lm_capabilities[{binding!r}].supports_response_schema must be a boolean")
|
|
55
|
+
dialect = declaration.get("dialect", CHAT_COMPLETIONS)
|
|
56
|
+
if dialect not in _DIALECTS:
|
|
57
|
+
raise LMConfigurationError(
|
|
58
|
+
f"lm_capabilities[{binding!r}].dialect must be one of {sorted(_DIALECTS)} (got {dialect!r})"
|
|
59
|
+
)
|
|
60
|
+
if dialect == SYSTEM_ONE and (params or schema):
|
|
61
|
+
raise LMConfigurationError(
|
|
62
|
+
f"lm_capabilities[{binding!r}] is a System One binding, which takes no sampling "
|
|
63
|
+
"parameters and no response schema"
|
|
64
|
+
)
|
|
41
65
|
if schema and "response_format" not in params:
|
|
42
66
|
raise LMConfigurationError(
|
|
43
67
|
f"lm_capabilities[{binding!r}] requires response_format in supported_params for schema output"
|
|
44
68
|
)
|
|
45
|
-
|
|
69
|
+
hints: LMCapabilities = {"supported_params": list(dict.fromkeys(params)), "supports_response_schema": schema}
|
|
70
|
+
if dialect == SYSTEM_ONE:
|
|
71
|
+
hints["dialect"] = SYSTEM_ONE
|
|
72
|
+
result[binding] = hints
|
|
46
73
|
return result
|
|
47
74
|
|
|
48
75
|
|
|
@@ -122,6 +149,11 @@ def configure_lm(
|
|
|
122
149
|
)
|
|
123
150
|
_merge(_names, _name(lm), hints)
|
|
124
151
|
return
|
|
152
|
+
_declare(lm, hints)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _declare(lm: Any, hints: dict[str, Any]) -> None:
|
|
156
|
+
"""Attach a declaration to one LM object, shared by every binding made from it."""
|
|
125
157
|
model = _model(lm)
|
|
126
158
|
declared = _object_declaration(lm)
|
|
127
159
|
if declared is not None and declared != (model, hints):
|
|
@@ -131,6 +163,38 @@ def configure_lm(
|
|
|
131
163
|
_objects.append(lm)
|
|
132
164
|
|
|
133
165
|
|
|
166
|
+
class SystemOne:
|
|
167
|
+
"""A System One decision model, for a run's ``lms``.
|
|
168
|
+
|
|
169
|
+
A System One model answers a set of questions about a state instead of continuing a
|
|
170
|
+
conversation: TypeSafe's Jev, or another endpoint serving the same schema. A hosted
|
|
171
|
+
judge built on ``dspy.experimental.TypeSafe`` runs against the run's System One
|
|
172
|
+
binding without naming a key or a URL::
|
|
173
|
+
|
|
174
|
+
cmpnd.optimize(..., lms={"judge": cmpnd.SystemOne("typesafe/jev-latest")})
|
|
175
|
+
|
|
176
|
+
``model`` is a platform model id: the prefix names the organization's sealed
|
|
177
|
+
credential, and the rest is the model at that credential's endpoint. Writing the
|
|
178
|
+
binding this way is the only thing that makes it a System One binding; a model name
|
|
179
|
+
on its own never does.
|
|
180
|
+
"""
|
|
181
|
+
|
|
182
|
+
def __init__(self, model: object):
|
|
183
|
+
if not isinstance(model, str) or not model.strip():
|
|
184
|
+
raise ValueError(f"cmpnd.SystemOne needs a model id such as 'typesafe/jev-latest' (got {model!r})")
|
|
185
|
+
credential, _, name = model.strip().partition("/")
|
|
186
|
+
if not credential or not name:
|
|
187
|
+
raise ValueError(
|
|
188
|
+
f"cmpnd.SystemOne({model!r}) names no credential: write the model as "
|
|
189
|
+
"'<credential>/<model>', for example 'typesafe/jev-latest'"
|
|
190
|
+
)
|
|
191
|
+
self.model = model.strip()
|
|
192
|
+
_declare(self, {"supported_params": [], "supports_response_schema": False, "dialect": SYSTEM_ONE})
|
|
193
|
+
|
|
194
|
+
def __repr__(self) -> str:
|
|
195
|
+
return f"cmpnd.SystemOne({self.model!r})"
|
|
196
|
+
|
|
197
|
+
|
|
134
198
|
def restore(declarations: object) -> None:
|
|
135
199
|
"""Load model-pinned declarations from a packaged artifact."""
|
|
136
200
|
if not isinstance(declarations, dict):
|
|
@@ -462,6 +462,23 @@ def _lms_config(config: dict[str, Any], lms: Any) -> None:
|
|
|
462
462
|
if isinstance(lm, str):
|
|
463
463
|
declared[name] = {"model": lm}
|
|
464
464
|
continue
|
|
465
|
+
from .lm_configuration import SystemOne
|
|
466
|
+
|
|
467
|
+
if isinstance(lm, SystemOne):
|
|
468
|
+
# A System One model takes no sampling, and its declaration (which says it
|
|
469
|
+
# is one) travels in the bundle rather than on the wire.
|
|
470
|
+
declared[name] = {"model": lm.model}
|
|
471
|
+
continue
|
|
472
|
+
if getattr(lm, "supports_decision_requests", False):
|
|
473
|
+
# dspy.experimental.TypeSafe, or a client like it: it carries a key and a URL
|
|
474
|
+
# rather than the name of a sealed credential, so the platform has nothing to
|
|
475
|
+
# resolve. Duck-typed so this module never imports dspy at scope.
|
|
476
|
+
raise ValueError(
|
|
477
|
+
f"{subject} is a {type(lm).__name__} client, which names a key and a URL rather "
|
|
478
|
+
f"than a credential. Bind the model with cmpnd.SystemOne, for example "
|
|
479
|
+
f"lms={{{name!r}: cmpnd.SystemOne('typesafe/jev-latest')}}; a judge that builds "
|
|
480
|
+
f"its own {type(lm).__name__} runs against that binding when hosted"
|
|
481
|
+
)
|
|
465
482
|
model = _lm_model(lm, subject)
|
|
466
483
|
_refuse_credential_on(lm, model, subject)
|
|
467
484
|
entry: dict[str, Any] = {"model": model}
|
|
@@ -73,6 +73,27 @@ endpoint accept JSON schemas. Undeclared bindings keep prompt-only JSON. These
|
|
|
73
73
|
hints do not add a format the provider cannot accept or change local DSPy's
|
|
74
74
|
capability lookup.
|
|
75
75
|
|
|
76
|
+
### System One judges
|
|
77
|
+
|
|
78
|
+
A System One model answers a set of questions about a state instead of continuing
|
|
79
|
+
a conversation. TypeSafe's Jev is one, and other endpoints serve the same schema.
|
|
80
|
+
Declare one for a run with `cmpnd.SystemOne`:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
cmpnd.optimize(..., lms={"judge": cmpnd.SystemOne("typesafe/jev-latest")})
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The prefix names the organization's sealed credential and the rest names the
|
|
87
|
+
model at that credential's endpoint. Writing the binding this way is the only
|
|
88
|
+
thing that makes it a System One binding; a model name never does. The binding
|
|
89
|
+
carries no sampling, and its declaration travels in the bundle like any other.
|
|
90
|
+
|
|
91
|
+
Hosted, a judge built on `dspy.experimental.TypeSafe` runs unchanged. The kernel
|
|
92
|
+
replaces `TypeSafe` with a client that spends the run's System One binding, so
|
|
93
|
+
the judge's own model, key and base URL are not used. With several System One
|
|
94
|
+
bindings, the judge picks one through `lms[name]`. A `TypeSafe` object passed in
|
|
95
|
+
`lms` is refused, because it names a key and a URL rather than a credential.
|
|
96
|
+
|
|
76
97
|
## The callback
|
|
77
98
|
|
|
78
99
|
`python-sdk/cmpnd/callback.py` implements `CmpndCallback`, which satisfies `dspy.utils.callback.BaseCallback` without inheriting it — `callback.py` must import on the standard library alone (`tests/test_import_weight.py`), so it cannot import dspy to get a base class. DSPy calls its hook methods synchronously on whatever thread is executing the module. The callback owns no long-lived state beyond a per-call span builder map; everything trace/span related flows through context vars (see next section).
|
|
@@ -5,7 +5,7 @@ name = "cmpnd"
|
|
|
5
5
|
# it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
|
|
6
6
|
# hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
|
|
7
7
|
# this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.45.0"
|
|
9
9
|
description = "DSPy observability and deployment SDK for cmpnd"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.11"
|
|
@@ -199,6 +199,48 @@ def feedback_metric(gold, pred, trace=None, pred_name=None, pred_trace=None, lms
|
|
|
199
199
|
return dspy.Prediction(score=0.0, feedback=f"answered nothing; the expected answer is {want!r}{verdict}")
|
|
200
200
|
|
|
201
201
|
|
|
202
|
+
def system_one_metric(gold, pred, trace=None, pred_name=None, pred_trace=None, lms=None):
|
|
203
|
+
""":func:`feedback_metric`, judging the way a System One judge written for a laptop does.
|
|
204
|
+
|
|
205
|
+
The judge builds its own ``TypeSafe`` with a model name and no binding. Hosted, the
|
|
206
|
+
kernel replaces that client with one that spends the run's System One binding, which
|
|
207
|
+
the run declares with ``cmpnd.SystemOne``. The verdict goes into the feedback and never
|
|
208
|
+
into the score, for :func:`feedback_metric`'s reason.
|
|
209
|
+
|
|
210
|
+
``dspy.experimental`` needs DSPy 3.4, which the kernel runs and this client may not, so
|
|
211
|
+
the judge's imports are inside the branch that only a hosted run reaches.
|
|
212
|
+
"""
|
|
213
|
+
import dspy
|
|
214
|
+
|
|
215
|
+
got = str(getattr(pred, "answer", "") or "").strip().lower()
|
|
216
|
+
want = str(getattr(gold, "answer", "") or "").strip().lower()
|
|
217
|
+
|
|
218
|
+
verdict = ""
|
|
219
|
+
if lms is not None:
|
|
220
|
+
from dspy.experimental import Noul, TypeSafe
|
|
221
|
+
|
|
222
|
+
class Right(dspy.Signature):
|
|
223
|
+
"""Decide whether the answer is right."""
|
|
224
|
+
|
|
225
|
+
question: str = dspy.InputField()
|
|
226
|
+
answer: str = dspy.InputField()
|
|
227
|
+
right: Noul = dspy.OutputField(desc="is the answer right")
|
|
228
|
+
|
|
229
|
+
judge = dspy.Predict(Right)
|
|
230
|
+
judge.lm = TypeSafe("jev-latest")
|
|
231
|
+
judged = judge(question=str(getattr(gold, "question", "")), answer=got)
|
|
232
|
+
verdict = f" (the judge put {judged.right.probability:.2f} on right)"
|
|
233
|
+
|
|
234
|
+
if got == want:
|
|
235
|
+
return dspy.Prediction(score=1.0, feedback=f"exactly right{verdict}")
|
|
236
|
+
if got:
|
|
237
|
+
return dspy.Prediction(
|
|
238
|
+
score=0.25,
|
|
239
|
+
feedback=(f"answered {got!r} but the expected answer is {want!r}; answer in one lowercase word{verdict}"),
|
|
240
|
+
)
|
|
241
|
+
return dspy.Prediction(score=0.0, feedback=f"answered nothing; the expected answer is {want!r}{verdict}")
|
|
242
|
+
|
|
243
|
+
|
|
202
244
|
def refuse_every_proposal(candidate, reflective_dataset, components):
|
|
203
245
|
"""A proposer that never works, which a run must not report as "no improvement".
|
|
204
246
|
|
|
@@ -353,6 +395,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
|
|
353
395
|
),
|
|
354
396
|
)
|
|
355
397
|
p.add_argument("--timeout", type=float, default=1800.0, help="seconds to wait for the run to settle")
|
|
398
|
+
p.add_argument(
|
|
399
|
+
"--judge",
|
|
400
|
+
choices=("chat", "system-one"),
|
|
401
|
+
default="chat",
|
|
402
|
+
help=(
|
|
403
|
+
"what the metric's judge binding is: a chat model, or a System One model it judges "
|
|
404
|
+
"through DSPy's System One client"
|
|
405
|
+
),
|
|
406
|
+
)
|
|
356
407
|
p.add_argument(
|
|
357
408
|
"--metric-names-an-ungranted-binding",
|
|
358
409
|
action="store_true",
|
|
@@ -378,6 +429,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
378
429
|
program = build_program()
|
|
379
430
|
started_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
380
431
|
metric = ungranted_binding_metric if args.metric_names_an_ungranted_binding else feedback_metric
|
|
432
|
+
if args.judge == "system-one":
|
|
433
|
+
metric = system_one_metric
|
|
381
434
|
|
|
382
435
|
print(f"deploying to {endpoint}", flush=True)
|
|
383
436
|
deployed = cmpnd.deploy(
|
|
@@ -394,6 +447,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
394
447
|
|
|
395
448
|
examples = [dspy.Example(**row).with_inputs("question") for row in EXAMPLES]
|
|
396
449
|
proposer, lms = declared_proposer(args.proposer)
|
|
450
|
+
if args.judge == "system-one":
|
|
451
|
+
# The same model the chat judge names, declared as System One: the run's sealed
|
|
452
|
+
# credential serves both protocols, and the dialect picks the endpoint.
|
|
453
|
+
lms = {**lms, JUDGE_BINDING: cmpnd.SystemOne(JUDGE_MODEL)}
|
|
397
454
|
print(
|
|
398
455
|
f"optimizing with a budget of {args.metric_calls} metric calls, "
|
|
399
456
|
f"proposer={args.proposer}, bindings={sorted(lms)}, "
|
|
@@ -404,6 +461,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
404
461
|
"started_at": started_at,
|
|
405
462
|
"parent_deployment_id": str(deployed),
|
|
406
463
|
"proposer": args.proposer,
|
|
464
|
+
"judge": args.judge,
|
|
407
465
|
}
|
|
408
466
|
|
|
409
467
|
def optimize() -> Any:
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""A DSPy ``TypeSafe`` decision call is an LM call.
|
|
2
|
+
|
|
3
|
+
``dspy.clients.typesafe.TypeSafe`` asks a System One model questions about a state. It
|
|
4
|
+
spends tokens and records an LM-shaped history entry, but it is not a ``dspy.BaseLM``, so
|
|
5
|
+
DSPy's ``with_callbacks`` hands its calls to ``on_module_start``. The callback recognizes
|
|
6
|
+
the client and records an LM span with a usage observation, as it does for a chat call.
|
|
7
|
+
|
|
8
|
+
Each case runs DSPy's own ``TypeSafe.__call__`` under DSPy's own callback dispatch. Only
|
|
9
|
+
the network step is a stand-in: a ``typesafe_sdk`` module whose client answers from a
|
|
10
|
+
fixture.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import sys
|
|
16
|
+
from types import ModuleType, SimpleNamespace
|
|
17
|
+
from typing import Any
|
|
18
|
+
from unittest import mock
|
|
19
|
+
|
|
20
|
+
import pytest
|
|
21
|
+
|
|
22
|
+
from cmpnd import models
|
|
23
|
+
from cmpnd.configuration import configure
|
|
24
|
+
from cmpnd.context import clear_context
|
|
25
|
+
|
|
26
|
+
dspy = pytest.importorskip("dspy")
|
|
27
|
+
typesafe = pytest.importorskip("dspy.clients.typesafe", reason="this DSPy has no TypeSafe decision client")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _RecordingExporter:
|
|
31
|
+
def __init__(self) -> None:
|
|
32
|
+
self.spans: list[Any] = []
|
|
33
|
+
|
|
34
|
+
def export_span(self, span: Any) -> None:
|
|
35
|
+
self.spans.append(span)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class _Client:
|
|
39
|
+
"""`typesafe_sdk.TypeSafeClient`, answering every turn with one decision.
|
|
40
|
+
|
|
41
|
+
The test environment does not install `typesafe-sdk`, so this copies the
|
|
42
|
+
signatures of 0.7.5's constructor and `system_one` by hand. Neither takes
|
|
43
|
+
`**kwargs`, so a call the real client would reject raises here too.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
constructed: list[dict[str, Any]] = []
|
|
47
|
+
|
|
48
|
+
def __init__(
|
|
49
|
+
self,
|
|
50
|
+
*,
|
|
51
|
+
api_key: str | None = None,
|
|
52
|
+
model: str | None = None,
|
|
53
|
+
retry: Any = None,
|
|
54
|
+
timeout: Any = None,
|
|
55
|
+
headers: Any = None,
|
|
56
|
+
transport: Any = None,
|
|
57
|
+
http_client: Any = None,
|
|
58
|
+
base_url: str | None = None,
|
|
59
|
+
compression: Any = None,
|
|
60
|
+
) -> None:
|
|
61
|
+
_Client.constructed.append({"api_key": api_key, "model": model, "timeout": timeout, "base_url": base_url})
|
|
62
|
+
|
|
63
|
+
def __enter__(self) -> _Client:
|
|
64
|
+
return self
|
|
65
|
+
|
|
66
|
+
def __exit__(self, *_exc: object) -> None:
|
|
67
|
+
return None
|
|
68
|
+
|
|
69
|
+
def system_one(
|
|
70
|
+
self,
|
|
71
|
+
state: Any,
|
|
72
|
+
questions: Any,
|
|
73
|
+
*,
|
|
74
|
+
model: str | None = None,
|
|
75
|
+
assets: Any = None,
|
|
76
|
+
retry: Any = None,
|
|
77
|
+
timeout: Any = None,
|
|
78
|
+
extra_headers: Any = None,
|
|
79
|
+
extra_body: Any = None,
|
|
80
|
+
response_model: Any = None,
|
|
81
|
+
) -> Any:
|
|
82
|
+
return SimpleNamespace(
|
|
83
|
+
model="jev-1.14.0",
|
|
84
|
+
usage=SimpleNamespace(input_tokens=31, output_tokens=2),
|
|
85
|
+
answers={"grounded": SimpleNamespace(noul="yes", confidence=0.9)},
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@pytest.fixture(autouse=True)
|
|
90
|
+
def _sdk_configured():
|
|
91
|
+
configure(api_key="test_key", endpoint="http://localhost:8000")
|
|
92
|
+
_Client.constructed.clear()
|
|
93
|
+
stand_in = ModuleType("typesafe_sdk")
|
|
94
|
+
stand_in.TypeSafeClient = _Client # pyright: ignore[reportAttributeAccessIssue]
|
|
95
|
+
with mock.patch.dict(sys.modules, {"typesafe_sdk": stand_in}):
|
|
96
|
+
yield
|
|
97
|
+
clear_context()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _spans_of(client: Any) -> list[Any]:
|
|
101
|
+
from cmpnd.callback import CmpndCallback
|
|
102
|
+
|
|
103
|
+
with mock.patch("cmpnd.exporter.ensure_exporter_started", autospec=True):
|
|
104
|
+
callback = CmpndCallback()
|
|
105
|
+
recorder = _RecordingExporter()
|
|
106
|
+
with (
|
|
107
|
+
mock.patch("cmpnd.exporter.get_exporter", autospec=True, return_value=recorder),
|
|
108
|
+
dspy.context(callbacks=[callback]),
|
|
109
|
+
):
|
|
110
|
+
answers = client(state="the reply cites its source", questions={"grounded": "Is it grounded?"})
|
|
111
|
+
assert answers == {"grounded": {"noul": "yes", "confidence": 0.9}}
|
|
112
|
+
return recorder.spans
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_a_decision_call_records_an_lm_span_with_its_usage():
|
|
116
|
+
spans = _spans_of(typesafe.TypeSafe("jev-latest", cache=False))
|
|
117
|
+
|
|
118
|
+
assert [span.span_type for span in spans] == [models.SpanType.LM_CALL]
|
|
119
|
+
assert [kwargs["model"] for kwargs in _Client.constructed] == ["jev-latest"]
|
|
120
|
+
observation = spans[0].to_api_dict()["lm_usage"]
|
|
121
|
+
assert observation["usage_reported"] is True
|
|
122
|
+
assert observation["prompt_tokens"] == 31
|
|
123
|
+
assert observation["completion_tokens"] == 2
|
|
124
|
+
assert observation["model_reported"] == "jev-1.14.0"
|
|
125
|
+
# The protocol it spoke, which the token field names alone would call chat.
|
|
126
|
+
assert observation["usage_dialect"] == "system_one"
|
|
127
|
+
# System One serves nothing from a cache, so the whole prompt is fresh input, as
|
|
128
|
+
# credentialed egress records the same call.
|
|
129
|
+
assert observation["prompt_inclusion"] == "excludes_cache_reads"
|
|
130
|
+
assert observation["fresh_input_tokens"] == 31
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_a_decision_call_keeps_the_brokers_join_key():
|
|
134
|
+
"""A brokered client puts the broker's `egress_call_id` on its response, and the
|
|
135
|
+
observation carries it, so serve joins the span to egress's record of the call."""
|
|
136
|
+
|
|
137
|
+
class Brokered(typesafe.TypeSafe):
|
|
138
|
+
@staticmethod
|
|
139
|
+
def _response(response: Any) -> dict[str, Any]:
|
|
140
|
+
return {**typesafe.TypeSafe._response(response), "egress_call_id": "egc_judge"}
|
|
141
|
+
|
|
142
|
+
spans = _spans_of(Brokered("jev-latest", cache=False))
|
|
143
|
+
|
|
144
|
+
assert spans[0].to_api_dict()["lm_usage"]["egress_call_id"] == "egc_judge"
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""``cmpnd.SystemOne`` — a run's binding for a System One decision model.
|
|
2
|
+
|
|
3
|
+
A System One model (TypeSafe's Jev, or the same schema served by another gateway) is
|
|
4
|
+
not a chat model: a hosted judge built on ``dspy.experimental.TypeSafe`` asks it a set of
|
|
5
|
+
questions about a state. ``cmpnd.SystemOne(model)`` in ``lms`` is the one way a run says
|
|
6
|
+
a binding is one, and these tests hold the two things it writes: the binding the
|
|
7
|
+
dispatch resolves a credential from, and the declaration the kernel builds its client
|
|
8
|
+
from.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import pytest
|
|
14
|
+
|
|
15
|
+
import cmpnd
|
|
16
|
+
from cmpnd import lm_configuration as lc
|
|
17
|
+
from cmpnd import optimization
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.fixture(autouse=True)
|
|
21
|
+
def reset():
|
|
22
|
+
lc.reset()
|
|
23
|
+
yield
|
|
24
|
+
lc.reset()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _config(lms):
|
|
28
|
+
config: dict = {}
|
|
29
|
+
optimization._lms_config(config, lms)
|
|
30
|
+
return config
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_a_system_one_binding_travels_as_its_model_alone():
|
|
34
|
+
"""The dispatch resolves the credential from the model and nothing else, so the wire
|
|
35
|
+
entry is a plain binding: no sampling, because System One takes none."""
|
|
36
|
+
assert _config({"judge": cmpnd.SystemOne("lunaroute/djev")}) == {"lms": {"judge": {"model": "lunaroute/djev"}}}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_its_declaration_says_system_one():
|
|
40
|
+
judge = cmpnd.SystemOne("typesafe/jev-latest")
|
|
41
|
+
assert lc.resolve({"judge": judge}) == {
|
|
42
|
+
"judge": {
|
|
43
|
+
"model": "typesafe/jev-latest",
|
|
44
|
+
"supported_params": [],
|
|
45
|
+
"supports_response_schema": False,
|
|
46
|
+
"dialect": "system_one",
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_a_chat_declaration_carries_no_dialect():
|
|
52
|
+
"""Chat is what a declaration without the key already means, so a chat binding's
|
|
53
|
+
declaration is byte-for-byte what it was before System One existed."""
|
|
54
|
+
lm = type("LM", (), {"model": "openai/gpt-4o"})()
|
|
55
|
+
lc.configure_lm(lm, supported_params=["response_format"])
|
|
56
|
+
assert "dialect" not in lc.resolve({"critic": lm})["critic"]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def test_a_restored_system_one_declaration_round_trips():
|
|
60
|
+
declaration = {
|
|
61
|
+
"model": "lunaroute/clef-flash",
|
|
62
|
+
"supported_params": [],
|
|
63
|
+
"supports_response_schema": False,
|
|
64
|
+
"dialect": "system_one",
|
|
65
|
+
}
|
|
66
|
+
lc.restore({"judge": declaration})
|
|
67
|
+
assert lc.resolve({"judge": "lunaroute/clef-flash"}) == {"judge": declaration}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@pytest.mark.parametrize(
|
|
71
|
+
"declaration, match",
|
|
72
|
+
[
|
|
73
|
+
({"dialect": "embedding"}, "dialect"),
|
|
74
|
+
({"dialect": "typesafe"}, "dialect"),
|
|
75
|
+
({"dialect": "system_one", "supported_params": ["temperature"]}, "takes no"),
|
|
76
|
+
({"dialect": "system_one", "supports_response_schema": True}, "takes no"),
|
|
77
|
+
],
|
|
78
|
+
)
|
|
79
|
+
def test_an_unusable_dialect_declaration_is_refused(declaration, match):
|
|
80
|
+
with pytest.raises(lc.LMConfigurationError, match=match):
|
|
81
|
+
lc.restore({"judge": {"model": "typesafe/jev-latest", **declaration}})
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@pytest.mark.parametrize("model", ["jev-latest", "", " ", "typesafe/", 42])
|
|
85
|
+
def test_a_system_one_model_must_name_its_credential(model):
|
|
86
|
+
"""The prefix names the credential the platform spends, so a bare model name has no
|
|
87
|
+
credential to resolve and is refused where it is written."""
|
|
88
|
+
with pytest.raises(ValueError, match="SystemOne"):
|
|
89
|
+
cmpnd.SystemOne(model)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class TypeSafe:
|
|
93
|
+
"""Stands in for ``dspy.experimental.TypeSafe``, which DSPy releases before 3.4 lack.
|
|
94
|
+
The refusal reads the decision-client flag, as DSPy's own ``Predict`` does."""
|
|
95
|
+
|
|
96
|
+
supports_decision_requests = True
|
|
97
|
+
|
|
98
|
+
def __init__(self, model):
|
|
99
|
+
self.model = model
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_a_decision_client_in_lms_points_at_system_one():
|
|
103
|
+
"""A ``TypeSafe`` carries a key and a URL, not a credential name, so it cannot be a
|
|
104
|
+
hosted binding. The refusal says what to write instead."""
|
|
105
|
+
with pytest.raises(ValueError, match=r"cmpnd\.SystemOne"):
|
|
106
|
+
_config({"judge": TypeSafe("jev-latest")})
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|