cmpnd 0.65.0__tar.gz → 0.66.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cmpnd-0.65.0 → cmpnd-0.66.0}/PKG-INFO +15 -5
- {cmpnd-0.65.0 → cmpnd-0.66.0}/README.md +14 -4
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/optimizations.py +28 -3
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/client.py +24 -1
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/optimization.py +22 -4
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/cli.md +15 -7
- {cmpnd-0.65.0 → cmpnd-0.66.0}/pyproject.toml +1 -1
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_continuation.py +34 -4
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_optimize_continuation.py +25 -11
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_optimizations.py +46 -2
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize.py +35 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/uv.lock +1 -1
- {cmpnd-0.65.0 → cmpnd-0.66.0}/.gitignore +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/AGENTS.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/CLAUDE.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_assessments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_authored_import.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_deployments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_exporter_pool.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_gepa_patch.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_http.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_program_patch.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_rlm_patch.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_runtime_tools.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_timespec.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_traces.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/_valsets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/authored.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/callback.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/api_client.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/admin.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/assessments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/auth.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/datasets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/deployments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/evals.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/login.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/org.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/provider_credentials.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/tool_servers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/top.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/traces.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/commands/valsets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/credentials.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/deprecation.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/pagination.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/shell.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/trace_render.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/tree_help.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/cli/verbs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/configuration.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/context.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/credentials.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/dataset_sync.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/datasets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/decorators.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/deployment.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/deployment_metadata.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/deployment_runs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/eval_handler.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/execution.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/exporter.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/exporter_http.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/helpers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/identity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/imaging.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/decode.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/encode.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/hash.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/program.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/reflection.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/types.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/ir/xxh64.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/lm_configuration.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/mcp_servers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/models.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/named_programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/optimizers/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/optimizers/gepa.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/optimizers/gepa_callback.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/optimizers/resume.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/optimizers/tracker.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/packaging.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/references.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/retry.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/cmpnd/traces.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/2026-04-02-code-review.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/client.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/custom-proposer.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/plans/2026-04-05-train-val-dataset-capture-design.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/plans/2026-04-05-train-val-dataset-capture.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/reading-traces.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/docs/sdk-instrumentation.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/examples/generalizing_proposer.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/examples/local_ollama.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/LINEAGE.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/README.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/bodies.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/client_floor.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/exporter_http.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/program_names.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/protocols.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/py.typed +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/request_log.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/retry.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/cmpnd_ingest/trace_facts.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/ingest/pyproject.toml +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/audit-sdist.sh +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/capture_review_run.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/check-pip-install.sh +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/check_release_artifacts.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/fixtures/review_run.json.gz +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/gepa_resume_bug.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/install-test-env.sh +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/release-stress.sh +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/require_version_bump.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/seed_review_run.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/selfhost_compat.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/scripts/test-in-linux.sh +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/stubs/dspy/__init__.pyi +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/stubs/dspy/primitives/__init__.pyi +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/stubs/dspy/primitives/prediction.pyi +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/stubs/dspy/utils/__init__.pyi +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/stubs/dspy/utils/callback.pyi +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/stubs/dspy/utils/dummies.pyi +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_credential_route.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_deployment_signature_helpers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_exporter_helpers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_fake_clock.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_identity_helpers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_resolve_refs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/_traced_run.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/conftest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/committee_task.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/conftest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/continuation_task.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/data/committee_train.ndjson.gz +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/data/committee_val.ndjson.gz +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/fake_lm.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/helpers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/target_capabilities.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_assessments_cli.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_capture_prompts.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_composite_module.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_dataset_auto_creation.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_deploy_execute.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_deploy_pandas.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_deploy_programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_edge_cases.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_example_hash.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_gepa_optimization.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_interpreter_spans.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_lm_accuracy.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_module_tracing.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimization_lineage.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimization_metric_identity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_authored_metric.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_custom_proposer.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_failures.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_loop.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_review_committee.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_optimize_server_owned_completion.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_parallel_export.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_project_attribution.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_react_tool_spans.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_save_load.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_trace_structure.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_traces_read.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_user_attribution.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/e2e/test_valsets_cli.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/01_predict_qa.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/02_predict_renamed_field.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/03_chain_of_thought_qa.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/04_predict_richer_types.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/05_composite_two_predicts.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/06_multichaincomparison.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/07_program_of_thought.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/08_opaque_module.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/09_predict_pydantic_field.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/10_react.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/11_refine.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/12_retrieve.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/13_knn.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/14_parallel.json +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/identity_vectors/README.md +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/ingest_conformance.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/authored_optimize/judge.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/authored_optimize/optimize_hosted.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/conftest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/hosted_mcp_tool_optimize_probe.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/hosted_optimize_probe.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/optimization_run_probe.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/test_cli_deploy_smoketest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/test_logs_renders_traces.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/test_optimization_run.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/test_optimizing_progress.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/integration/test_unhappy_path.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/__main__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/drain.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/ingest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/latency_transport.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/rlm_shape.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/rlm_size_report.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/sweep.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/test_rlm_shape.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/load/test_rlm_size_report.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/conftest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/seed_parity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_auth.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_datasets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_deployments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_evals.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_health.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_modules.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_optimizations.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_signatures.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_trace_streaming.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/parity/test_traces.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/span_reference_fixture.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/_provider.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/_readiness.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/_reliability.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/_result.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/mint_bedrock_token.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_bedrock_converse.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_deploy_programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_multi_client_stress.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_optimize_client_committee.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_optimize_loop.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_optimize_loop_wasm.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_optimize_stress.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_provider_roles.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_readiness.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_real_provider_twins.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_reliability.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_reliability_classify.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_rlm_wasm.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/stress/test_trace_search.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_adapter_reentrancy.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_assessments_api.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_authored.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_authored_binding_credentials.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_authored_bundle_collisions.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_authored_metric_contract.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_authored_named_models.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_authored_optimize.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_callback.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_candidate_lineage.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/conftest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_admin.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_api_client.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_assessments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_auth.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_credentials.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_datasets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_deployment_runs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_deployments.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_evals.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_login.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_main.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_org.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_pagination.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_parity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_programs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_provider_credentials.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_schema_sample.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_shell.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_tool_servers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_top.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_traces.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_traces_program_and_module.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_traces_search_wire.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_tree_help.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cli/test_valsets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_client.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_client_floor.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_client_program_names.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_clustering/__init__.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_clustering/conftest.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_clustering/test_edge_cases.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_clustering/test_hash_consistency.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_clustering/test_type_conversion.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_cmp551_grouping_identity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_configuration.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_context.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_context_propagation.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_credential_kwargs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_credential_selection.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_credential_values.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_dataset_polling.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_dataset_sync.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_datasets.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_decision_client_spans.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_decorators.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_deploy.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_deploy_mcp_tool_references.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_deploy_program_id.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_deployment_metadata.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_deployment_runs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_deployments_read.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_eval.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_eval_example_attribution.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_eval_start.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_eval_trace_grouping.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_exception_paths.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_execute.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_execute_run_identity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_exporter.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_exporter_program_names.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_exporter_queue.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_exporter_real.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_fake_lm.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_gepa_callback.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_gepa_integration.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_gepa_output_normalization.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_gepa_resume.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_identity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_imaging.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_import_weight.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_ingest_conformance.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_ingest_second_producer.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_integration.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_interpreter_callback.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_ir_encoder_properties.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_ir_golden_vectors.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_ir_roundtrip.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_lm15_native_calls.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_lm_configuration.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_lm_usage_shapes.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_mcp_tools.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_models.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_multi_client_stress_readiness.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimization_progress.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimization_start.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize_budget.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize_lift.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize_lms.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize_proposer.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize_reflection_lm.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_optimize_terminal_failure.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_packaging.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_packaging_callable_source.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_packaging_mcp_tools.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_packaging_program_source.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_poll_until.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_program_id_deprecation.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_program_identity.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_program_lineage_persist.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_program_names_in_deploy.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_programs_create.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_project_payload.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_real_provider_twins_readiness.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_reflector_properties.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_reflector_smoke.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_release_stress_script.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_replaced_names.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_repro_reactv2_bugs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_require_version_bump.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_retry.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_reversible_failures.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_rlm_patch.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_run_attribution_paths.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_runtime_tools.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_schemas.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_seal_lm_credential.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_seed_review_run.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_signature_inputs.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_span_references.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_span_sent_once.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_system_one.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_trace_context_concurrency.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_trace_fact_wire_contract.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_trace_facts.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_trace_render.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_traces_read.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_tracked_gepa.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_tracker_enrich.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_truncation.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_upload_auth_headers.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_valsets_api.py +0 -0
- {cmpnd-0.65.0 → cmpnd-0.66.0}/tests/test_xxh64.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cmpnd
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.66.0
|
|
4
4
|
Summary: DSPy observability and deployment SDK for cmpnd
|
|
5
5
|
Project-URL: Homepage, https://cmpnd.ai
|
|
6
6
|
Author-email: cmpnd <hello@cmpnd.ai>
|
|
@@ -526,8 +526,17 @@ for candidate in sdk.list_candidates(best.optimization_run_id): # read each one
|
|
|
526
526
|
other = sdk.continue_optimization(best, candidate="ca_...", auto="light")
|
|
527
527
|
```
|
|
528
528
|
|
|
529
|
+
`model=` runs the candidate on another student model, and `credential=` names
|
|
530
|
+
the credential that model spends. A loop over models is a model sweep:
|
|
531
|
+
|
|
532
|
+
```python
|
|
533
|
+
for model in ["openai/gpt-4o-mini", "openai/gpt-4.1-nano"]:
|
|
534
|
+
swept = sdk.continue_optimization(best, model=model, auto="light")
|
|
535
|
+
print(model, swept.score)
|
|
536
|
+
```
|
|
537
|
+
|
|
529
538
|
A refusal raises `cmpnd.OptimizeError`, whose `error_code` says why, such as
|
|
530
|
-
`not_completed` or `
|
|
539
|
+
`not_completed`, `unknown_candidate` or `no_credential`.
|
|
531
540
|
|
|
532
541
|
## Command line
|
|
533
542
|
|
|
@@ -654,9 +663,10 @@ global config. Use it as a context manager or as a plain object.
|
|
|
654
663
|
found. It raises `cmpnd.OptimizeError` or `cmpnd.OptimizeFailed` on failure.
|
|
655
664
|
- `list_candidates(run_id)` returns a run's candidates as dicts, each with its
|
|
656
665
|
`ca_` id, score, parent and instructions.
|
|
657
|
-
- `continue_optimization(run, candidate=None, ...)`
|
|
658
|
-
from one candidate of a completed run, with that
|
|
659
|
-
and
|
|
666
|
+
- `continue_optimization(run, candidate=None, model=None, credential=None, ...)`
|
|
667
|
+
runs a new optimization from one candidate of a completed run, with that
|
|
668
|
+
run's data and a new budget, optionally on another student model, and returns
|
|
669
|
+
its best `DeployedProgram`.
|
|
660
670
|
- `list_traces(**filters)` and `get_trace(trace_id)` read traces with the
|
|
661
671
|
client's credentials. They flush the client's traces first.
|
|
662
672
|
- `close()` flushes the client's traces and closes the client. A closed
|
|
@@ -475,8 +475,17 @@ for candidate in sdk.list_candidates(best.optimization_run_id): # read each one
|
|
|
475
475
|
other = sdk.continue_optimization(best, candidate="ca_...", auto="light")
|
|
476
476
|
```
|
|
477
477
|
|
|
478
|
+
`model=` runs the candidate on another student model, and `credential=` names
|
|
479
|
+
the credential that model spends. A loop over models is a model sweep:
|
|
480
|
+
|
|
481
|
+
```python
|
|
482
|
+
for model in ["openai/gpt-4o-mini", "openai/gpt-4.1-nano"]:
|
|
483
|
+
swept = sdk.continue_optimization(best, model=model, auto="light")
|
|
484
|
+
print(model, swept.score)
|
|
485
|
+
```
|
|
486
|
+
|
|
478
487
|
A refusal raises `cmpnd.OptimizeError`, whose `error_code` says why, such as
|
|
479
|
-
`not_completed` or `
|
|
488
|
+
`not_completed`, `unknown_candidate` or `no_credential`.
|
|
480
489
|
|
|
481
490
|
## Command line
|
|
482
491
|
|
|
@@ -603,9 +612,10 @@ global config. Use it as a context manager or as a plain object.
|
|
|
603
612
|
found. It raises `cmpnd.OptimizeError` or `cmpnd.OptimizeFailed` on failure.
|
|
604
613
|
- `list_candidates(run_id)` returns a run's candidates as dicts, each with its
|
|
605
614
|
`ca_` id, score, parent and instructions.
|
|
606
|
-
- `continue_optimization(run, candidate=None, ...)`
|
|
607
|
-
from one candidate of a completed run, with that
|
|
608
|
-
and
|
|
615
|
+
- `continue_optimization(run, candidate=None, model=None, credential=None, ...)`
|
|
616
|
+
runs a new optimization from one candidate of a completed run, with that
|
|
617
|
+
run's data and a new budget, optionally on another student model, and returns
|
|
618
|
+
its best `DeployedProgram`.
|
|
609
619
|
- `list_traces(**filters)` and `get_trace(trace_id)` read traces with the
|
|
610
620
|
client's credentials. They flush the client's traces first.
|
|
611
621
|
- `close()` flushes the client's traces and closes the client. A closed
|
|
@@ -93,7 +93,7 @@ def register(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
93
93
|
cont = sub.add_parser(
|
|
94
94
|
"continue",
|
|
95
95
|
help="Run a new optimization from one candidate of a completed run. It reuses that run's data, models "
|
|
96
|
-
"and credentials, and takes a new budget",
|
|
96
|
+
"and credentials, and takes a new budget. --model runs the candidate on another student model",
|
|
97
97
|
)
|
|
98
98
|
cont.add_argument("id", help="Completed optimization run ID (op_...)")
|
|
99
99
|
cont.add_argument(
|
|
@@ -102,6 +102,19 @@ def register(subparsers: _SubParsersAction[ArgumentParser]) -> None:
|
|
|
102
102
|
help="The ca_ id of the candidate to start from, as `cmpnd optimizations candidates ID` lists them "
|
|
103
103
|
"(default: the run's best)",
|
|
104
104
|
)
|
|
105
|
+
cont.add_argument(
|
|
106
|
+
"--model",
|
|
107
|
+
default=None,
|
|
108
|
+
help="The student model the new run spends, as a deploy records it, for example openai/gpt-4o-mini "
|
|
109
|
+
"(default: the completed run's). Only the program's own model changes",
|
|
110
|
+
)
|
|
111
|
+
cont.add_argument(
|
|
112
|
+
"--credential",
|
|
113
|
+
metavar="NAME",
|
|
114
|
+
default=None,
|
|
115
|
+
help="The provider credential the student model spends (default: the completed run's). "
|
|
116
|
+
"`cmpnd credentials list` shows the names",
|
|
117
|
+
)
|
|
105
118
|
# The budget flags are `cmpnd optimize`'s, and the SDK refuses a run with none or two,
|
|
106
119
|
# so both commands answer a missing budget the same way.
|
|
107
120
|
cont.add_argument(
|
|
@@ -258,17 +271,23 @@ def handle_continue(client: api_client.ApiClient, args: Any) -> Any:
|
|
|
258
271
|
"""Run a new optimization from one candidate of a completed run, as `cmpnd optimize` runs one.
|
|
259
272
|
|
|
260
273
|
The new run reuses the completed run's data and configuration and takes a new budget.
|
|
261
|
-
|
|
262
|
-
|
|
274
|
+
`--model` and `--credential` run the candidate on another student model. The command
|
|
275
|
+
names the candidate it starts from, waits for the run, and prints the best program as
|
|
276
|
+
`cmpnd optimize` does. `--async` returns the queued run instead.
|
|
263
277
|
"""
|
|
264
278
|
from ... import optimization
|
|
265
279
|
|
|
280
|
+
credential_name = args.credential
|
|
281
|
+
if credential_name is not None:
|
|
282
|
+
credential_name = top._CredentialLookup(client)(f"--credential {credential_name}", credential_name).name
|
|
266
283
|
cfg = top.sdk_config(client)
|
|
267
284
|
try:
|
|
268
285
|
started = optimization.start_continuation(
|
|
269
286
|
args.id,
|
|
270
287
|
cfg=cfg,
|
|
271
288
|
candidate=args.candidate,
|
|
289
|
+
model=args.model,
|
|
290
|
+
credential_name=credential_name,
|
|
272
291
|
max_metric_calls=args.max_metric_calls,
|
|
273
292
|
auto=args.auto,
|
|
274
293
|
)
|
|
@@ -289,6 +308,11 @@ def handle_continue(client: api_client.ApiClient, args: Any) -> Any:
|
|
|
289
308
|
),
|
|
290
309
|
file=sys.stderr,
|
|
291
310
|
)
|
|
311
|
+
if args.model is not None or args.credential is not None:
|
|
312
|
+
print(
|
|
313
|
+
top._c(f"on {started.get('model')} with the credential {started.get('credential_name')}", "dim"),
|
|
314
|
+
file=sys.stderr,
|
|
315
|
+
)
|
|
292
316
|
print(top._c(f"optimizing with {started.get('optimizer')}…", "dim"), file=sys.stderr)
|
|
293
317
|
try:
|
|
294
318
|
result = optimization.wait_for_continuation(started, cfg=cfg)
|
|
@@ -301,6 +325,7 @@ def handle_continue(client: api_client.ApiClient, args: Any) -> Any:
|
|
|
301
325
|
"optimization_id": started.get("optimization_id"),
|
|
302
326
|
"continued_from_run_id": started.get("continued_from_run_id"),
|
|
303
327
|
"continued_from_candidate_id": started.get("continued_from_candidate_id"),
|
|
328
|
+
**{key: started[key] for key in ("model", "credential_name") if key in started},
|
|
304
329
|
},
|
|
305
330
|
)
|
|
306
331
|
return None
|
|
@@ -721,6 +721,8 @@ class Client:
|
|
|
721
721
|
run: Any,
|
|
722
722
|
*,
|
|
723
723
|
candidate: str | None = None,
|
|
724
|
+
model: str | None = None,
|
|
725
|
+
credential: credentials.Credential | None = None,
|
|
724
726
|
max_metric_calls: int | None = None,
|
|
725
727
|
auto: str | None = None,
|
|
726
728
|
poll_interval: float = 3.0,
|
|
@@ -735,12 +737,27 @@ class Client:
|
|
|
735
737
|
best = client.optimize_deployment(deployed, ..., max_metric_calls=120)
|
|
736
738
|
better = client.continue_optimization(best, max_metric_calls=120)
|
|
737
739
|
|
|
740
|
+
With ``model``, the new run is a model sweep: the candidate runs and optimizes on
|
|
741
|
+
that student model instead of the completed run's. A loop over models compares
|
|
742
|
+
them on one task::
|
|
743
|
+
|
|
744
|
+
for model in ["openai/gpt-4o-mini", "openai/gpt-4.1-nano"]:
|
|
745
|
+
swept = client.continue_optimization(best, model=model, auto="light")
|
|
746
|
+
print(model, swept.score)
|
|
747
|
+
|
|
738
748
|
Args:
|
|
739
749
|
run: The completed run's ``op_`` id, or the ``DeployedProgram`` an
|
|
740
750
|
optimization returned.
|
|
741
751
|
candidate: The ``ca_`` id of the candidate to start from, as
|
|
742
752
|
``list_candidates`` and ``cmpnd optimizations candidates`` list it. The
|
|
743
753
|
default is the run's best.
|
|
754
|
+
model: The student model the new run spends, as a deploy records it, for
|
|
755
|
+
example ``"openai/gpt-4o-mini"``. The default is the completed run's.
|
|
756
|
+
Only the program's own model changes: a predictor with its own LM, the
|
|
757
|
+
run's ``lms`` and the reflection model stay as the completed run had
|
|
758
|
+
them.
|
|
759
|
+
credential: The credential the student model spends, from
|
|
760
|
+
``cmpnd.credentials.get(NAME)``. The default is the completed run's.
|
|
744
761
|
max_metric_calls: The budget as a ceiling on metric calls. Pass this or
|
|
745
762
|
``auto``, exactly one.
|
|
746
763
|
auto: The budget as ``"light"``, ``"medium"`` or ``"heavy"``.
|
|
@@ -754,11 +771,15 @@ class Client:
|
|
|
754
771
|
Raises:
|
|
755
772
|
ValueError: No budget or two budgets, or a ``DeployedProgram`` that no
|
|
756
773
|
optimization produced.
|
|
774
|
+
TypeError: ``credential`` is a string. Look a credential up with
|
|
775
|
+
``cmpnd.credentials.get``.
|
|
757
776
|
OptimizeError: The platform refused the continuation. Its ``error_code``
|
|
758
|
-
says why, for example ``not_completed`` or
|
|
777
|
+
says why, for example ``not_completed``, ``unknown_candidate`` or
|
|
778
|
+
``no_credential``.
|
|
759
779
|
OptimizeFailed: The new run transitioned to ``status="failed"``.
|
|
760
780
|
"""
|
|
761
781
|
self._check_active()
|
|
782
|
+
credential_name = credentials.name_of(credential, "cmpnd.Client.continue_optimization(credential=)")
|
|
762
783
|
from . import execution, optimization
|
|
763
784
|
|
|
764
785
|
run_id = run.optimization_run_id if isinstance(run, execution.DeployedProgram) else run
|
|
@@ -770,6 +791,8 @@ class Client:
|
|
|
770
791
|
run_id,
|
|
771
792
|
cfg=self._config,
|
|
772
793
|
candidate=candidate,
|
|
794
|
+
model=model,
|
|
795
|
+
credential_name=credential_name,
|
|
773
796
|
max_metric_calls=max_metric_calls,
|
|
774
797
|
auto=auto,
|
|
775
798
|
poll_interval=poll_interval,
|
|
@@ -978,20 +978,28 @@ def start_continuation(
|
|
|
978
978
|
*,
|
|
979
979
|
cfg: configuration.Config,
|
|
980
980
|
candidate: str | None = None,
|
|
981
|
+
model: str | None = None,
|
|
982
|
+
credential_name: str | None = None,
|
|
981
983
|
max_metric_calls: int | None = None,
|
|
982
984
|
auto: str | None = None,
|
|
983
985
|
) -> dict[str, Any]:
|
|
984
986
|
"""Start a run from one candidate of the completed run ``run_id``, and return serve's answer.
|
|
985
987
|
|
|
986
988
|
The answer names the new run (``optimization_id``), its deployment and optimizer, the
|
|
987
|
-
run and ``ca_`` candidate it continues,
|
|
988
|
-
|
|
989
|
-
|
|
989
|
+
run and ``ca_`` candidate it continues, the candidate's score and instructions, and
|
|
990
|
+
the ``model`` and ``credential_name`` the run spends. Serve deploys the candidate and
|
|
991
|
+
reuses the parent's evaluation data and configuration, so the request carries only
|
|
992
|
+
the candidate, the budget and an optional student model and credential, and nothing
|
|
993
|
+
is uploaded.
|
|
990
994
|
"""
|
|
991
995
|
body: dict[str, Any] = {}
|
|
992
996
|
_apply_budget(body, max_metric_calls=max_metric_calls, auto=auto, config_budgets=False)
|
|
993
997
|
if candidate is not None:
|
|
994
998
|
body["candidate_id"] = candidate
|
|
999
|
+
if model is not None:
|
|
1000
|
+
body["model"] = model
|
|
1001
|
+
if credential_name is not None:
|
|
1002
|
+
body["credential_name"] = credential_name
|
|
995
1003
|
headers = {"X-API-Key": cfg.api_key, "Content-Type": "application/json"}
|
|
996
1004
|
what = f"continue optimization {run_id}"
|
|
997
1005
|
with _http.new_client(timeout=_HTTP_TIMEOUT_SECONDS) as client:
|
|
@@ -1052,6 +1060,8 @@ def continue_optimization(
|
|
|
1052
1060
|
*,
|
|
1053
1061
|
cfg: configuration.Config,
|
|
1054
1062
|
candidate: str | None = None,
|
|
1063
|
+
model: str | None = None,
|
|
1064
|
+
credential_name: str | None = None,
|
|
1055
1065
|
max_metric_calls: int | None = None,
|
|
1056
1066
|
auto: str | None = None,
|
|
1057
1067
|
poll_interval: float = 3.0,
|
|
@@ -1061,7 +1071,15 @@ def continue_optimization(
|
|
|
1061
1071
|
|
|
1062
1072
|
``cmpnd.Client.continue_optimization`` calls this and documents the arguments.
|
|
1063
1073
|
"""
|
|
1064
|
-
started = start_continuation(
|
|
1074
|
+
started = start_continuation(
|
|
1075
|
+
run_id,
|
|
1076
|
+
cfg=cfg,
|
|
1077
|
+
candidate=candidate,
|
|
1078
|
+
model=model,
|
|
1079
|
+
credential_name=credential_name,
|
|
1080
|
+
max_metric_calls=max_metric_calls,
|
|
1081
|
+
auto=auto,
|
|
1082
|
+
)
|
|
1065
1083
|
return wait_for_continuation(started, cfg=cfg, poll_interval=poll_interval, timeout=timeout)
|
|
1066
1084
|
|
|
1067
1085
|
|
|
@@ -759,12 +759,19 @@ so nothing is uploaded.
|
|
|
759
759
|
candidate's instructions.
|
|
760
760
|
- `--candidate CANDIDATE_ID` names the candidate by its `ca_` id. The default is
|
|
761
761
|
the run's best.
|
|
762
|
+
- `--model MODEL` runs the candidate on another student model, as a deploy
|
|
763
|
+
records it, such as `openai/gpt-4o-mini`. This is a model sweep. Only the
|
|
764
|
+
program's own model changes. A predictor with its own LM, the run's named LMs
|
|
765
|
+
and the reflection model stay as the completed run had them.
|
|
766
|
+
- `--credential NAME` names the credential the student model spends. The
|
|
767
|
+
default is the completed run's. The command looks the name up before it
|
|
768
|
+
starts the run, as `cmpnd optimize` does.
|
|
762
769
|
- The command runs the optimization as `cmpnd optimize` does. It names the
|
|
763
770
|
candidate it starts from on stderr, with its score and the start of its
|
|
764
771
|
instructions, then waits. It prints the best deployment's id on stdout and
|
|
765
772
|
its score on stderr. `--json` prints `best_deployment_id` and `best_score`,
|
|
766
|
-
with the new run's `optimization_id
|
|
767
|
-
from.
|
|
773
|
+
with the new run's `optimization_id`, the run and candidate it continued
|
|
774
|
+
from, and the `model` and `credential_name` it spends.
|
|
768
775
|
- `--async` prints the queued run as JSON and returns at once. Follow it with
|
|
769
776
|
`cmpnd optimizations show NEW_RUN_ID`, which shows `continued_from_run_id` and
|
|
770
777
|
`continued_from_candidate_id`. Showing the completed run lists its
|
|
@@ -773,11 +780,12 @@ so nothing is uploaded.
|
|
|
773
780
|
refused with a reason.
|
|
774
781
|
|
|
775
782
|
In Python, `cmpnd.Client.list_candidates(run_id)` returns the candidates as
|
|
776
|
-
dicts, and `cmpnd.Client.continue_optimization(run, candidate=None,
|
|
777
|
-
max_metric_calls=None, auto=None)` runs the continuation and
|
|
778
|
-
`DeployedProgram`, as `optimize_deployment` does. `run` is an
|
|
779
|
-
`DeployedProgram` an optimization returned, whose
|
|
780
|
-
its run. `candidate` is a `ca_` id.
|
|
783
|
+
dicts, and `cmpnd.Client.continue_optimization(run, candidate=None, model=None,
|
|
784
|
+
credential=None, max_metric_calls=None, auto=None)` runs the continuation and
|
|
785
|
+
returns its best `DeployedProgram`, as `optimize_deployment` does. `run` is an
|
|
786
|
+
`op_` id or the `DeployedProgram` an optimization returned, whose
|
|
787
|
+
`optimization_run_id` names its run. `candidate` is a `ca_` id. `model` is a
|
|
788
|
+
model string, and `credential` is a `cmpnd.credentials.get(NAME)` result.
|
|
781
789
|
|
|
782
790
|
An org's plan may cap how many hosted optimizations run at once. A run started
|
|
783
791
|
over the cap waits in serve's queue and starts when one of the org's runs
|
|
@@ -5,7 +5,7 @@ name = "cmpnd"
|
|
|
5
5
|
# it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
|
|
6
6
|
# hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
|
|
7
7
|
# this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.66.0"
|
|
9
9
|
description = "DSPy observability and deployment SDK for cmpnd"
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.11"
|
|
@@ -6,6 +6,10 @@ observable on fake models: the seed scores 0.0 and gepa's first proposal, "Answe
|
|
|
6
6
|
word `correct`.", scores 1.0 (``fake-provider/README.md``). So a continuation's own first
|
|
7
7
|
candidate shows which candidate seeded it: it holds that candidate's text and its score.
|
|
8
8
|
|
|
9
|
+
A model sweep is a continuation on another student model. fake-provider's model
|
|
10
|
+
``cmpnd-fake/deaf`` skips the instruction rule, so the proposal that scores 1.0 on the
|
|
11
|
+
parent's model scores 0.0 on it, which shows the student model changed.
|
|
12
|
+
|
|
9
13
|
The parent runs once per module. Each test continues from it.
|
|
10
14
|
"""
|
|
11
15
|
|
|
@@ -25,6 +29,9 @@ pytestmark = [pytest.mark.e2e, needs_an_optimizer]
|
|
|
25
29
|
# Room for the seed's evaluation on the valset and one proposal with its evaluation.
|
|
26
30
|
BUDGET_METRIC_CALLS = 30
|
|
27
31
|
|
|
32
|
+
#: fake-provider's model that answers the filler whatever the instruction says.
|
|
33
|
+
DEAF_MODEL = f"{fake_lm.PROVIDER}/deaf"
|
|
34
|
+
|
|
28
35
|
|
|
29
36
|
@pytest.fixture(scope="module")
|
|
30
37
|
def parent():
|
|
@@ -54,19 +61,22 @@ def _idx(candidate: dict[str, Any]) -> int:
|
|
|
54
61
|
return candidate["idx"]
|
|
55
62
|
|
|
56
63
|
|
|
57
|
-
def _continue(
|
|
58
|
-
|
|
64
|
+
def _continue(
|
|
65
|
+
run_id: str, candidate: dict[str, Any], model: str | None = None
|
|
66
|
+
) -> tuple[cmpnd.DeployedProgram, list[dict[str, Any]]]:
|
|
67
|
+
"""Continue ``run_id`` from ``candidate`` and wait; return the new run's best program and its candidates."""
|
|
59
68
|
with platform_client() as client:
|
|
60
69
|
result = client.continue_optimization(
|
|
61
70
|
run_id,
|
|
62
71
|
candidate=candidate["candidate_id"],
|
|
72
|
+
model=model,
|
|
63
73
|
max_metric_calls=BUDGET_METRIC_CALLS,
|
|
64
74
|
poll_interval=2.0,
|
|
65
75
|
timeout=300.0,
|
|
66
76
|
)
|
|
67
77
|
assert isinstance(result, cmpnd.DeployedProgram)
|
|
68
78
|
assert result.optimization_run_id and result.optimization_run_id != run_id
|
|
69
|
-
return result
|
|
79
|
+
return result, client.list_candidates(result.optimization_run_id)
|
|
70
80
|
|
|
71
81
|
|
|
72
82
|
class TestTheParentRun:
|
|
@@ -81,7 +91,8 @@ class TestTheParentRun:
|
|
|
81
91
|
class TestContinuingFromACandidate:
|
|
82
92
|
def test_a_continuation_from_the_proposal_starts_from_its_text_and_score(self, parent, server_client):
|
|
83
93
|
run_id, _, proposal = parent
|
|
84
|
-
|
|
94
|
+
result, candidates = _continue(run_id, proposal)
|
|
95
|
+
child = result.optimization_run_id
|
|
85
96
|
|
|
86
97
|
# The continuation's own seed is the proposal, and a perfect seed leaves gepa
|
|
87
98
|
# nothing to propose.
|
|
@@ -101,6 +112,19 @@ class TestContinuingFromACandidate:
|
|
|
101
112
|
assert (first["instructions"], first["aggregate_score"]) == (seed["instructions"], 0.0)
|
|
102
113
|
|
|
103
114
|
|
|
115
|
+
class TestAModelSweep:
|
|
116
|
+
def test_a_sweep_runs_the_candidate_on_the_named_model(self, parent, server_client):
|
|
117
|
+
run_id, _, proposal = parent
|
|
118
|
+
result, candidates = _continue(run_id, proposal, model=DEAF_MODEL)
|
|
119
|
+
|
|
120
|
+
# The proposal's text scores 1.0 on the parent's model and 0.0 on the deaf one.
|
|
121
|
+
first = min(candidates, key=_idx)
|
|
122
|
+
assert (first["instructions"], first["aggregate_score"]) == (proposal["instructions"], 0.0)
|
|
123
|
+
assert server_client.get_deployment(result.deployment_id)["litellm_model"] == DEAF_MODEL
|
|
124
|
+
detail = server_client.get_optimization(result.optimization_run_id)
|
|
125
|
+
assert detail["continued_from_candidate_id"] == proposal["candidate_id"]
|
|
126
|
+
|
|
127
|
+
|
|
104
128
|
class TestARefusedContinuationStartsNothing:
|
|
105
129
|
@pytest.mark.parametrize(
|
|
106
130
|
"candidate",
|
|
@@ -114,6 +138,12 @@ class TestARefusedContinuationStartsNothing:
|
|
|
114
138
|
assert raised.value.error_code == "unknown_candidate"
|
|
115
139
|
assert run_id in str(raised.value)
|
|
116
140
|
|
|
141
|
+
def test_an_empty_model_is_refused(self, parent):
|
|
142
|
+
run_id, _, _ = parent
|
|
143
|
+
with platform_client() as client, pytest.raises(cmpnd.OptimizeError) as raised:
|
|
144
|
+
client.continue_optimization(run_id, model=" ", max_metric_calls=BUDGET_METRIC_CALLS)
|
|
145
|
+
assert raised.value.error_code == "invalid_model"
|
|
146
|
+
|
|
117
147
|
def test_a_run_that_does_not_exist_is_refused(self):
|
|
118
148
|
with platform_client() as client, pytest.raises(cmpnd.OptimizeError) as raised:
|
|
119
149
|
client.continue_optimization("op_AAAAAAAAAAAA", max_metric_calls=BUDGET_METRIC_CALLS)
|
|
@@ -1,14 +1,16 @@
|
|
|
1
|
-
"""Stress: continue a completed optimization from its best candidate
|
|
1
|
+
"""Stress: continue a completed optimization from its best candidate onto another model.
|
|
2
2
|
|
|
3
3
|
The real-model twin of ``tests/e2e/test_optimize_continuation.py``. It runs the same task,
|
|
4
4
|
``continuation_task``, whose gold answer to every question is the word ``correct``. A real
|
|
5
5
|
reflector can learn that rule only from the metric's feedback, so its proposals score
|
|
6
6
|
above the seed. The test optimizes a deployment, picks the best candidate from
|
|
7
|
-
``list_candidates``, and continues the completed run from it with a new budget
|
|
7
|
+
``list_candidates``, and continues the completed run from it with a new budget on another
|
|
8
|
+
student model. That continuation is a model sweep. It names no credential, so it keeps
|
|
9
|
+
the parent's.
|
|
8
10
|
|
|
9
|
-
The assertions are about the platform, and the scores are logged. A
|
|
10
|
-
|
|
11
|
-
Real models do not repeat their scores exactly, so no score is asserted.
|
|
11
|
+
The assertions are about the platform, and the scores are logged. A sweep starts from the
|
|
12
|
+
chosen candidate's instructions, runs on the named model, completes, and records where it
|
|
13
|
+
came from. Real models do not repeat their scores exactly, so no score is asserted.
|
|
12
14
|
|
|
13
15
|
Env (stress convention, as ``test_optimize_loop_wasm.py``):
|
|
14
16
|
- PLATFORM_API_KEY: cmpnd server API key, an org owner's or one with ``credentials:write``
|
|
@@ -41,6 +43,9 @@ LM_KEY = _provider.LM_KEY
|
|
|
41
43
|
# student, and a reflector whose output ceiling fits a reflection turn.
|
|
42
44
|
STUDENT = _provider.deployed_model("student", committee_task.STUDENT_3B)
|
|
43
45
|
REFLECTION = _provider.deployed_model("reflection", "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0")
|
|
46
|
+
# The model the sweep moves the chosen candidate onto: the stress lane's fast model, from
|
|
47
|
+
# the same provider as the student, so the parent's credential serves it.
|
|
48
|
+
SWEPT = _provider.deployed_model("stress", "bedrock/us.amazon.nova-micro-v1:0")
|
|
44
49
|
|
|
45
50
|
# Room for the seed's evaluation on the six-question valset and a few proposals, in each
|
|
46
51
|
# of the two runs.
|
|
@@ -60,9 +65,9 @@ def _client() -> cmpnd.Client:
|
|
|
60
65
|
return cmpnd.Client(api_key=PLATFORM_API_KEY, endpoint=PLATFORM_ENDPOINT)
|
|
61
66
|
|
|
62
67
|
|
|
63
|
-
def
|
|
68
|
+
def _read(path: str) -> dict[str, Any]:
|
|
64
69
|
resp = httpx.get(
|
|
65
|
-
f"{PLATFORM_ENDPOINT}/api/v1/
|
|
70
|
+
f"{PLATFORM_ENDPOINT}/api/v1/{path}",
|
|
66
71
|
headers={"X-API-Key": PLATFORM_API_KEY or ""},
|
|
67
72
|
timeout=60.0,
|
|
68
73
|
)
|
|
@@ -70,6 +75,10 @@ def _optimization(run_id: str) -> dict[str, Any]:
|
|
|
70
75
|
return resp.json()
|
|
71
76
|
|
|
72
77
|
|
|
78
|
+
def _optimization(run_id: str) -> dict[str, Any]:
|
|
79
|
+
return _read(f"optimizations/{run_id}")
|
|
80
|
+
|
|
81
|
+
|
|
73
82
|
def _describe(candidates: list[dict[str, Any]]) -> str:
|
|
74
83
|
return "; ".join(
|
|
75
84
|
f"{c['candidate_id']} #{c['idx']} {c['aggregate_score']:.3f} {c['instructions']}" for c in candidates
|
|
@@ -85,11 +94,11 @@ def _idx(candidate: dict[str, Any]) -> int:
|
|
|
85
94
|
|
|
86
95
|
|
|
87
96
|
class TestContinuationOnRealModels:
|
|
88
|
-
def
|
|
89
|
-
with tolerate_external_outage("
|
|
97
|
+
def test_a_sweep_starts_from_the_best_candidate_on_the_named_model_and_records_it(self):
|
|
98
|
+
with tolerate_external_outage("model sweep of a gepa run (real LM)"):
|
|
90
99
|
# The student and the reflector may name different providers, so each prefix
|
|
91
100
|
# gets its own credential. The program's LM carries neither a key nor a base.
|
|
92
|
-
_provider.seal_deployed_credential(STUDENT, REFLECTION)
|
|
101
|
+
_provider.seal_deployed_credential(STUDENT, REFLECTION, SWEPT)
|
|
93
102
|
lm = dspy.LM(STUDENT, temperature=0.0, max_tokens=400, cache=False)
|
|
94
103
|
program = cmpnd.Program(
|
|
95
104
|
f"test_optimize_continuation-{uuid.uuid4().hex[:8]}",
|
|
@@ -120,6 +129,7 @@ class TestContinuationOnRealModels:
|
|
|
120
129
|
result = client.continue_optimization(
|
|
121
130
|
parent,
|
|
122
131
|
candidate=chosen["candidate_id"],
|
|
132
|
+
model=SWEPT,
|
|
123
133
|
max_metric_calls=BUDGET_METRIC_CALLS,
|
|
124
134
|
poll_interval=5.0,
|
|
125
135
|
timeout=1800.0,
|
|
@@ -146,8 +156,12 @@ class TestContinuationOnRealModels:
|
|
|
146
156
|
max(c["aggregate_score"] for c in child_candidates),
|
|
147
157
|
)
|
|
148
158
|
|
|
149
|
-
# The
|
|
159
|
+
# The sweep ran on the named model, and its best deployment records it.
|
|
150
160
|
detail = _optimization(child)
|
|
161
|
+
assert detail["model_name"] == SWEPT, f"sweep {child} ran on {detail['model_name']}, not {SWEPT}"
|
|
162
|
+
assert _read(f"deployments/{result.deployment_id}")["litellm_model"] == SWEPT
|
|
163
|
+
|
|
164
|
+
# The lineage reads both ways.
|
|
151
165
|
assert (detail["status"], detail["continued_from_run_id"], detail["continued_from_candidate_id"]) == (
|
|
152
166
|
"completed",
|
|
153
167
|
parent,
|
|
@@ -8,6 +8,7 @@ from unittest import mock
|
|
|
8
8
|
|
|
9
9
|
import pytest
|
|
10
10
|
|
|
11
|
+
import cmpnd
|
|
11
12
|
from cmpnd.cli.api_client import CliError
|
|
12
13
|
from cmpnd.cli.commands import optimizations
|
|
13
14
|
from cmpnd.cli.commands.optimizations import (
|
|
@@ -218,11 +219,11 @@ def _client_with_account(mock_client):
|
|
|
218
219
|
class TestOptimizationsContinue:
|
|
219
220
|
"""`continue` runs an optimization as `cmpnd optimize` does: it waits, and prints the same result."""
|
|
220
221
|
|
|
221
|
-
def _run(self, mock_client, argv):
|
|
222
|
+
def _run(self, mock_client, argv, started=_STARTED):
|
|
222
223
|
args = _build_parser().parse_args(["optimizations", "continue", "op_abc", *argv])
|
|
223
224
|
assert args.func is handle_continue
|
|
224
225
|
with (
|
|
225
|
-
mock.patch("cmpnd.optimization.start_continuation", autospec=True, return_value=
|
|
226
|
+
mock.patch("cmpnd.optimization.start_continuation", autospec=True, return_value=started) as m_start,
|
|
226
227
|
mock.patch("cmpnd.optimization.wait_for_continuation", autospec=True, return_value=_Result()) as m_wait,
|
|
227
228
|
):
|
|
228
229
|
returned = handle_continue(_client_with_account(mock_client), args)
|
|
@@ -254,6 +255,49 @@ class TestOptimizationsContinue:
|
|
|
254
255
|
"continued_from_candidate_id": "ca_U0IdawjhyJbz",
|
|
255
256
|
}
|
|
256
257
|
|
|
258
|
+
def test_a_sweep_names_its_model_and_looks_up_its_credential(self, mock_client, capsys):
|
|
259
|
+
"""`--model` makes the continuation a sweep, and `--credential` is looked up as `optimize` looks it up."""
|
|
260
|
+
bedrock = cmpnd.credentials._mint("bedrock", ("converse",), "https://bedrock-runtime.us-east-1.amazonaws.com")
|
|
261
|
+
swept = {**_STARTED, "model": "bedrock/us.amazon.nova-micro-v1:0", "credential_name": "bedrock"}
|
|
262
|
+
with mock.patch("cmpnd.credentials.get", autospec=True, return_value=bedrock) as m_get:
|
|
263
|
+
_, m_start, _ = self._run(
|
|
264
|
+
mock_client,
|
|
265
|
+
[
|
|
266
|
+
"--model",
|
|
267
|
+
"bedrock/us.amazon.nova-micro-v1:0",
|
|
268
|
+
"--credential",
|
|
269
|
+
"bedrock",
|
|
270
|
+
"--auto",
|
|
271
|
+
"light",
|
|
272
|
+
"--json",
|
|
273
|
+
],
|
|
274
|
+
started=swept,
|
|
275
|
+
)
|
|
276
|
+
assert m_get.call_args.args == ("bedrock",)
|
|
277
|
+
kwargs = m_start.call_args.kwargs
|
|
278
|
+
assert (kwargs["model"], kwargs["credential_name"]) == ("bedrock/us.amazon.nova-micro-v1:0", "bedrock")
|
|
279
|
+
out, err = capsys.readouterr()
|
|
280
|
+
assert "on bedrock/us.amazon.nova-micro-v1:0 with the credential bedrock" in err
|
|
281
|
+
summary = json.loads(out)
|
|
282
|
+
assert (summary["model"], summary["credential_name"]) == ("bedrock/us.amazon.nova-micro-v1:0", "bedrock")
|
|
283
|
+
|
|
284
|
+
def test_a_continuation_with_no_model_sends_none(self, mock_client):
|
|
285
|
+
_, m_start, _ = self._run(mock_client, ["--auto", "light"])
|
|
286
|
+
assert (m_start.call_args.kwargs["model"], m_start.call_args.kwargs["credential_name"]) == (None, None)
|
|
287
|
+
|
|
288
|
+
def test_an_unknown_credential_is_refused_before_the_continuation(self, mock_client):
|
|
289
|
+
args = _build_parser().parse_args(
|
|
290
|
+
["optimizations", "continue", "op_abc", "--credential", "bedrok", "--auto", "light"]
|
|
291
|
+
)
|
|
292
|
+
missing = cmpnd.credentials.CredentialNotFound("bedrok", ["bedrock"], "bedrock")
|
|
293
|
+
with (
|
|
294
|
+
mock.patch("cmpnd.credentials.get", autospec=True, side_effect=missing),
|
|
295
|
+
mock.patch("cmpnd.optimization.start_continuation", autospec=True) as m_start,
|
|
296
|
+
pytest.raises(CliError, match="--credential bedrok"),
|
|
297
|
+
):
|
|
298
|
+
handle_continue(_client_with_account(mock_client), args)
|
|
299
|
+
m_start.assert_not_called()
|
|
300
|
+
|
|
257
301
|
def test_async_returns_the_queued_run_without_waiting(self, mock_client):
|
|
258
302
|
returned, m_start, m_wait = self._run(mock_client, ["--auto", "light", "--async"])
|
|
259
303
|
assert returned == _STARTED
|
|
@@ -1265,6 +1265,41 @@ class TestContinueOptimization:
|
|
|
1265
1265
|
(request,) = wire.sent("POST")
|
|
1266
1266
|
assert request.url.path == "/api/v1/optimizations/op_parent/continue"
|
|
1267
1267
|
|
|
1268
|
+
def test_a_sweep_posts_its_model_and_credential(self, wire):
|
|
1269
|
+
"""A sweep is a continuation that names the student model and the credential it spends."""
|
|
1270
|
+
wire.answer(
|
|
1271
|
+
"POST", 201, {**self._ANSWER, "model": "bedrock/us.amazon.nova-micro-v1:0", "credential_name": "bedrock"}
|
|
1272
|
+
)
|
|
1273
|
+
wire.answer("GET", 200, self._DONE)
|
|
1274
|
+
bedrock = cmpnd.credentials._mint("bedrock", ("converse",), "https://bedrock-runtime.us-east-1.amazonaws.com")
|
|
1275
|
+
|
|
1276
|
+
with _no_sleep():
|
|
1277
|
+
_sdk().continue_optimization(
|
|
1278
|
+
"op_parent", model="bedrock/us.amazon.nova-micro-v1:0", credential=bedrock, max_metric_calls=30
|
|
1279
|
+
)
|
|
1280
|
+
(request,) = wire.sent("POST")
|
|
1281
|
+
assert json.loads(request.content) == {
|
|
1282
|
+
"max_metric_calls": 30,
|
|
1283
|
+
"model": "bedrock/us.amazon.nova-micro-v1:0",
|
|
1284
|
+
"credential_name": "bedrock",
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
def test_a_sweep_with_a_model_alone_sends_no_credential(self, wire):
|
|
1288
|
+
wire.answer("POST", 201, self._ANSWER)
|
|
1289
|
+
|
|
1290
|
+
cfg = cmpnd.configuration.Config(api_key="ct_client", endpoint="http://client.test")
|
|
1291
|
+
optimization.start_continuation("op_parent", cfg=cfg, model="openai/gpt-4o-mini", auto="light")
|
|
1292
|
+
(request,) = wire.sent("POST")
|
|
1293
|
+
assert json.loads(request.content) == {"auto": "light", "model": "openai/gpt-4o-mini"}
|
|
1294
|
+
|
|
1295
|
+
def test_a_sweep_refuses_a_credential_named_by_a_string(self, mock_httpx):
|
|
1296
|
+
client = _client(mock_httpx)
|
|
1297
|
+
with pytest.raises(TypeError, match="cmpnd.credentials.get"):
|
|
1298
|
+
_sdk().continue_optimization(
|
|
1299
|
+
"op_parent", model="openai/gpt-4o-mini", credential="openai", max_metric_calls=10
|
|
1300
|
+
) # type: ignore[arg-type]
|
|
1301
|
+
assert client.post.call_count == 0
|
|
1302
|
+
|
|
1268
1303
|
def test_a_listed_candidate_is_named_by_its_id(self, wire):
|
|
1269
1304
|
wire.answer("GET", 200, self._CANDIDATES)
|
|
1270
1305
|
wire.answer("POST", 201, self._ANSWER)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|