cmpnd 0.20.0__tar.gz → 0.22.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (307) hide show
  1. {cmpnd-0.20.0 → cmpnd-0.22.0}/PKG-INFO +1 -1
  2. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/callback.py +215 -44
  3. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/eval_handler.py +25 -5
  4. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/helpers.py +0 -28
  5. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/models.py +14 -20
  6. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/2026-04-02-code-review.md +0 -1
  7. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/sdk-instrumentation.md +3 -1
  8. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/bodies.py +0 -3
  9. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/exporter_http.py +1 -24
  10. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/protocols.py +1 -2
  11. {cmpnd-0.20.0 → cmpnd-0.22.0}/pyproject.toml +1 -1
  12. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/capture_review_run.py +0 -2
  13. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/seed_review_run.py +3 -12
  14. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/drain.py +1 -3
  15. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/latency_transport.py +0 -2
  16. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_traces.py +0 -28
  17. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_callback.py +11 -102
  18. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_eval.py +14 -8
  19. cmpnd-0.22.0/tests/test_eval_example_attribution.py +285 -0
  20. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_eval_trace_grouping.py +0 -4
  21. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_exporter.py +11 -10
  22. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_exporter_real.py +0 -23
  23. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_ingest_conformance.py +0 -1
  24. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_ingest_second_producer.py +1 -2
  25. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_integration.py +0 -9
  26. cmpnd-0.22.0/tests/test_lm_usage_shapes.py +411 -0
  27. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_project_payload.py +0 -8
  28. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_repro_reactv2_bugs.py +7 -6
  29. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_schemas.py +3 -11
  30. {cmpnd-0.20.0 → cmpnd-0.22.0}/todo.md +1 -1
  31. {cmpnd-0.20.0 → cmpnd-0.22.0}/uv.lock +31 -31
  32. cmpnd-0.20.0/tests/test_stats_export.py +0 -118
  33. {cmpnd-0.20.0 → cmpnd-0.22.0}/.gitignore +0 -0
  34. {cmpnd-0.20.0 → cmpnd-0.22.0}/CLAUDE.md +0 -0
  35. {cmpnd-0.20.0 → cmpnd-0.22.0}/README.md +0 -0
  36. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/__init__.py +0 -0
  37. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/_gepa_patch.py +0 -0
  38. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/_http.py +0 -0
  39. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/_program_patch.py +0 -0
  40. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/_rlm_patch.py +0 -0
  41. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/_timespec.py +0 -0
  42. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/__init__.py +0 -0
  43. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/api_client.py +0 -0
  44. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/__init__.py +0 -0
  45. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/admin.py +0 -0
  46. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/assessments.py +0 -0
  47. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/auth.py +0 -0
  48. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/datasets.py +0 -0
  49. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/deployments.py +0 -0
  50. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/evals.py +0 -0
  51. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/login.py +0 -0
  52. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/optimizations.py +0 -0
  53. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/org.py +0 -0
  54. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/programs.py +0 -0
  55. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/provider_credentials.py +0 -0
  56. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/signatures.py +0 -0
  57. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/tool_servers.py +0 -0
  58. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/top.py +0 -0
  59. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/traces.py +0 -0
  60. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/commands/valsets.py +0 -0
  61. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/credentials.py +0 -0
  62. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/deprecation.py +0 -0
  63. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/pagination.py +0 -0
  64. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/shell.py +0 -0
  65. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/trace_render.py +0 -0
  66. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/cli/verbs.py +0 -0
  67. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/configuration.py +0 -0
  68. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/context.py +0 -0
  69. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/dataset_sync.py +0 -0
  70. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/datasets.py +0 -0
  71. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/decorators.py +0 -0
  72. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/deployment.py +0 -0
  73. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/execution.py +0 -0
  74. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/exporter.py +0 -0
  75. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/exporter_http.py +0 -0
  76. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/identity.py +0 -0
  77. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/imaging.py +0 -0
  78. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/__init__.py +0 -0
  79. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/decode.py +0 -0
  80. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/encode.py +0 -0
  81. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/hash.py +0 -0
  82. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/program.py +0 -0
  83. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/reflection.py +0 -0
  84. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/types.py +0 -0
  85. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/ir/xxh64.py +0 -0
  86. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/mcp_servers.py +0 -0
  87. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/optimization.py +0 -0
  88. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/optimizers/__init__.py +0 -0
  89. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/optimizers/gepa.py +0 -0
  90. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/optimizers/gepa_callback.py +0 -0
  91. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/optimizers/resume.py +0 -0
  92. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/optimizers/tracker.py +0 -0
  93. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/packaging.py +0 -0
  94. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/retry.py +0 -0
  95. {cmpnd-0.20.0 → cmpnd-0.22.0}/cmpnd/traces.py +0 -0
  96. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/cli.md +0 -0
  97. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/custom-proposer.md +0 -0
  98. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/plans/2026-04-05-train-val-dataset-capture-design.md +0 -0
  99. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/plans/2026-04-05-train-val-dataset-capture.md +0 -0
  100. {cmpnd-0.20.0 → cmpnd-0.22.0}/docs/reading-traces.md +0 -0
  101. {cmpnd-0.20.0 → cmpnd-0.22.0}/examples/generalizing_proposer.py +0 -0
  102. {cmpnd-0.20.0 → cmpnd-0.22.0}/examples/local_ollama.py +0 -0
  103. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/LINEAGE.md +0 -0
  104. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/README.md +0 -0
  105. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/__init__.py +0 -0
  106. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/py.typed +0 -0
  107. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/request_log.py +0 -0
  108. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/cmpnd_ingest/retry.py +0 -0
  109. {cmpnd-0.20.0 → cmpnd-0.22.0}/ingest/pyproject.toml +0 -0
  110. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/audit-sdist.sh +0 -0
  111. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/check_release_artifacts.py +0 -0
  112. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/fixtures/review_run.json.gz +0 -0
  113. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/gepa_resume_bug.py +0 -0
  114. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/require_version_bump.py +0 -0
  115. {cmpnd-0.20.0 → cmpnd-0.22.0}/scripts/test-in-linux.sh +0 -0
  116. {cmpnd-0.20.0 → cmpnd-0.22.0}/stubs/dspy/__init__.pyi +0 -0
  117. {cmpnd-0.20.0 → cmpnd-0.22.0}/stubs/dspy/primitives/__init__.pyi +0 -0
  118. {cmpnd-0.20.0 → cmpnd-0.22.0}/stubs/dspy/primitives/prediction.pyi +0 -0
  119. {cmpnd-0.20.0 → cmpnd-0.22.0}/stubs/dspy/utils/__init__.pyi +0 -0
  120. {cmpnd-0.20.0 → cmpnd-0.22.0}/stubs/dspy/utils/callback.pyi +0 -0
  121. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/__init__.py +0 -0
  122. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/_exporter_helpers.py +0 -0
  123. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/_fake_clock.py +0 -0
  124. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/_identity_helpers.py +0 -0
  125. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/conftest.py +0 -0
  126. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/__init__.py +0 -0
  127. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/committee_task.py +0 -0
  128. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/conftest.py +0 -0
  129. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/data/committee_train.ndjson.gz +0 -0
  130. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/data/committee_val.ndjson.gz +0 -0
  131. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/fake_lm.py +0 -0
  132. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/helpers.py +0 -0
  133. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/target_capabilities.py +0 -0
  134. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_assessments_cli.py +0 -0
  135. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_capture_prompts.py +0 -0
  136. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_composite_module.py +0 -0
  137. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_dataset_auto_creation.py +0 -0
  138. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_deploy_execute.py +0 -0
  139. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_deploy_programs.py +0 -0
  140. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_edge_cases.py +0 -0
  141. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_example_hash.py +0 -0
  142. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_gepa_optimization.py +0 -0
  143. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_interpreter_spans.py +0 -0
  144. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_lm_accuracy.py +0 -0
  145. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_module_tracing.py +0 -0
  146. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_optimization_lineage.py +0 -0
  147. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_optimization_metric_identity.py +0 -0
  148. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_optimize_failures.py +0 -0
  149. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_optimize_loop.py +0 -0
  150. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_optimize_review_committee.py +0 -0
  151. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_optimize_server_owned_completion.py +0 -0
  152. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_parallel_export.py +0 -0
  153. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_project_attribution.py +0 -0
  154. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_react_tool_spans.py +0 -0
  155. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_save_load.py +0 -0
  156. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_trace_structure.py +0 -0
  157. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_traces_read.py +0 -0
  158. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_user_attribution.py +0 -0
  159. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/e2e/test_valsets_cli.py +0 -0
  160. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/01_predict_qa.json +0 -0
  161. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/02_predict_renamed_field.json +0 -0
  162. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/03_chain_of_thought_qa.json +0 -0
  163. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/04_predict_richer_types.json +0 -0
  164. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/05_composite_two_predicts.json +0 -0
  165. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/06_multichaincomparison.json +0 -0
  166. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/07_program_of_thought.json +0 -0
  167. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/08_opaque_module.json +0 -0
  168. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/09_predict_pydantic_field.json +0 -0
  169. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/10_react.json +0 -0
  170. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/11_refine.json +0 -0
  171. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/12_retrieve.json +0 -0
  172. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/13_knn.json +0 -0
  173. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/14_parallel.json +0 -0
  174. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/identity_vectors/README.md +0 -0
  175. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/ingest_conformance.py +0 -0
  176. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/__init__.py +0 -0
  177. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/conftest.py +0 -0
  178. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/hosted_optimize_probe.py +0 -0
  179. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/optimization_run_probe.py +0 -0
  180. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/test_cli_deploy_smoketest.py +0 -0
  181. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/test_logs_renders_traces.py +0 -0
  182. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/test_optimization_run.py +0 -0
  183. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/test_optimizing_progress.py +0 -0
  184. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/integration/test_unhappy_path.py +0 -0
  185. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/__init__.py +0 -0
  186. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/__main__.py +0 -0
  187. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/ingest.py +0 -0
  188. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/rlm_shape.py +0 -0
  189. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/sweep.py +0 -0
  190. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/load/test_rlm_shape.py +0 -0
  191. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/__init__.py +0 -0
  192. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/conftest.py +0 -0
  193. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/seed_parity.py +0 -0
  194. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_auth.py +0 -0
  195. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_datasets.py +0 -0
  196. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_deployments.py +0 -0
  197. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_evals.py +0 -0
  198. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_health.py +0 -0
  199. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_modules.py +0 -0
  200. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_optimizations.py +0 -0
  201. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_programs.py +0 -0
  202. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_signatures.py +0 -0
  203. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/parity/test_trace_streaming.py +0 -0
  204. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/__init__.py +0 -0
  205. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/_provider.py +0 -0
  206. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/_reliability.py +0 -0
  207. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/_result.py +0 -0
  208. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/mint_bedrock_token.py +0 -0
  209. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_deploy_programs.py +0 -0
  210. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_multi_client_stress.py +0 -0
  211. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_optimize_client_committee.py +0 -0
  212. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_optimize_loop.py +0 -0
  213. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_optimize_loop_wasm.py +0 -0
  214. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_optimize_stress.py +0 -0
  215. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_provider_roles.py +0 -0
  216. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_real_provider_twins.py +0 -0
  217. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_reliability.py +0 -0
  218. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_reliability_classify.py +0 -0
  219. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_rlm_wasm.py +0 -0
  220. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/stress/test_trace_search.py +0 -0
  221. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_candidate_lineage.py +0 -0
  222. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/__init__.py +0 -0
  223. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/conftest.py +0 -0
  224. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_admin.py +0 -0
  225. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_api_client.py +0 -0
  226. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_assessments.py +0 -0
  227. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_auth.py +0 -0
  228. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_credentials.py +0 -0
  229. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_datasets.py +0 -0
  230. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_deployments.py +0 -0
  231. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_evals.py +0 -0
  232. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_login.py +0 -0
  233. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_main.py +0 -0
  234. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_optimizations.py +0 -0
  235. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_org.py +0 -0
  236. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_pagination.py +0 -0
  237. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_parity.py +0 -0
  238. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_programs.py +0 -0
  239. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_provider_credentials.py +0 -0
  240. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_schema_sample.py +0 -0
  241. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_shell.py +0 -0
  242. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_signatures.py +0 -0
  243. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_tool_servers.py +0 -0
  244. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_top.py +0 -0
  245. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_traces.py +0 -0
  246. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_traces_search_wire.py +0 -0
  247. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cli/test_valsets.py +0 -0
  248. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_clustering/__init__.py +0 -0
  249. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_clustering/conftest.py +0 -0
  250. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_clustering/test_edge_cases.py +0 -0
  251. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_clustering/test_hash_consistency.py +0 -0
  252. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_clustering/test_type_conversion.py +0 -0
  253. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_cmp551_grouping_identity.py +0 -0
  254. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_configuration.py +0 -0
  255. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_context.py +0 -0
  256. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_context_propagation.py +0 -0
  257. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_dataset_polling.py +0 -0
  258. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_dataset_sync.py +0 -0
  259. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_datasets.py +0 -0
  260. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_decorators.py +0 -0
  261. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_deploy.py +0 -0
  262. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_deploy_mcp_tool_references.py +0 -0
  263. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_eval_start.py +0 -0
  264. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_exception_paths.py +0 -0
  265. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_execute.py +0 -0
  266. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_exporter_queue.py +0 -0
  267. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_fake_lm.py +0 -0
  268. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_gepa_callback.py +0 -0
  269. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_gepa_integration.py +0 -0
  270. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_gepa_resume.py +0 -0
  271. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_identity.py +0 -0
  272. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_imaging.py +0 -0
  273. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_import_weight.py +0 -0
  274. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_interpreter_callback.py +0 -0
  275. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_ir_encoder_properties.py +0 -0
  276. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_ir_golden_vectors.py +0 -0
  277. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_ir_roundtrip.py +0 -0
  278. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_mcp_tools.py +0 -0
  279. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_models.py +0 -0
  280. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimization_progress.py +0 -0
  281. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimization_start.py +0 -0
  282. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize.py +0 -0
  283. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize_budget.py +0 -0
  284. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize_lift.py +0 -0
  285. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize_lms.py +0 -0
  286. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize_proposer.py +0 -0
  287. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize_reflection_lm.py +0 -0
  288. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_optimize_terminal_failure.py +0 -0
  289. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_packaging.py +0 -0
  290. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_packaging_callable_source.py +0 -0
  291. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_packaging_mcp_tools.py +0 -0
  292. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_packaging_program_source.py +0 -0
  293. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_poll_until.py +0 -0
  294. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_program_lineage_persist.py +0 -0
  295. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_reflector_properties.py +0 -0
  296. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_reflector_smoke.py +0 -0
  297. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_require_version_bump.py +0 -0
  298. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_retry.py +0 -0
  299. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_reversible_failures.py +0 -0
  300. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_rlm_patch.py +0 -0
  301. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_seed_review_run.py +0 -0
  302. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_trace_render.py +0 -0
  303. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_traces_read.py +0 -0
  304. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_tracked_gepa.py +0 -0
  305. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_tracker_enrich.py +0 -0
  306. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_truncation.py +0 -0
  307. {cmpnd-0.20.0 → cmpnd-0.22.0}/tests/test_xxh64.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: cmpnd
3
- Version: 0.20.0
3
+ Version: 0.22.0
4
4
  Summary: DSPy observability and deployment SDK for cmpnd
5
5
  Project-URL: Homepage, https://cmpnd.ai
6
6
  Author-email: cmpnd <hello@cmpnd.ai>
@@ -91,31 +91,182 @@ def _dspy_declares_hook(name: str) -> bool:
91
91
  return callable(getattr(base, name, None))
92
92
 
93
93
 
94
- def _cached_prompt_tokens(usage: Any) -> int:
95
- """Read the provider-side prompt-cache read count off a LiteLLM usage dict.
96
-
97
- Providers report cached input tokens two ways: the OpenAI/LiteLLM-normalized
98
- ``prompt_tokens_details.cached_tokens``, or a native ``cache_read_input_tokens``
99
- (Anthropic). We normalize to a single count that is always a subset of
100
- ``prompt_tokens`` — in the LiteLLM-normalized usage the SDK captures,
101
- ``prompt_tokens`` already includes the cached reads, so the subset invariant
102
- holds. ``prompt_tokens_details`` may be a dict or a small object, so both are
103
- handled. Returns 0 when nothing cache-related is present.
94
+ def _usage_field(usage: Any, key: str) -> Any:
95
+ """Read ``key`` off a usage block, whichever kind of object it is.
96
+
97
+ A usage block reaches us as a plain dict (DSPy's legacy history stores
98
+ ``dict(response.usage)``), as a LiteLLM usage object that also answers
99
+ ``.get``, or as DSPy's typed ``LMUsage`` pydantic model, which answers only
100
+ attribute access. All three have to work, so all three are tried.
101
+ """
102
+ if usage is None:
103
+ return None
104
+ if isinstance(usage, dict):
105
+ return usage.get(key)
106
+ getter = getattr(usage, "get", None)
107
+ if callable(getter):
108
+ try:
109
+ return getter(key)
110
+ except (TypeError, KeyError):
111
+ pass
112
+ return getattr(usage, key, None)
113
+
114
+
115
+ def _usage_detail(usage: Any, container: str, key: str) -> Any:
116
+ """Read ``container.key`` off a usage block's nested detail object.
117
+
118
+ The detail blocks (``prompt_tokens_details`` and its Responses-API twin)
119
+ arrive as a dict or as a small object depending on the provider and the
120
+ LiteLLM version, so both are read.
104
121
  """
105
- if not usage:
106
- return 0
107
- details = usage.get("prompt_tokens_details") if hasattr(usage, "get") else None
108
- cached = None
122
+ details = _usage_field(usage, container)
109
123
  if isinstance(details, dict):
110
- cached = details.get("cached_tokens")
111
- elif details is not None:
112
- cached = getattr(details, "cached_tokens", None)
113
- if not cached and hasattr(usage, "get"):
114
- cached = usage.get("cache_read_input_tokens")
115
- try:
116
- return int(cached) if cached else 0
117
- except (TypeError, ValueError):
118
- return 0
124
+ return details.get(key)
125
+ if details is None:
126
+ return None
127
+ return getattr(details, key, None)
128
+
129
+
130
+ def _reported_count(*candidates: Any) -> int | None:
131
+ """Reduce one count's several spellings to the number the provider reported.
132
+
133
+ Returns the first nonzero spelling; ``0`` when a spelling reported zero and
134
+ no other reported a count; ``None`` when no spelling reported at all. The
135
+ distinction is the point: a provider that says nothing about reasoning
136
+ tokens and one that reports none of them are different facts. Absent is not
137
+ zero, and the two stay apart all the way to the UI — a count nobody measured
138
+ must never be presented as one that was.
139
+
140
+ Preferring a nonzero spelling over an earlier zero is what makes the
141
+ Anthropic pair work: LiteLLM fills ``prompt_tokens_details.cached_tokens``
142
+ with a 0 alongside a populated native ``cache_read_input_tokens``.
143
+ """
144
+ saw_zero = False
145
+ for candidate in candidates:
146
+ if candidate is None:
147
+ continue
148
+ try:
149
+ value = int(candidate)
150
+ except (TypeError, ValueError):
151
+ continue
152
+ if value:
153
+ return value
154
+ saw_zero = True
155
+ return 0 if saw_zero else None
156
+
157
+
158
+ def _normalize_usage(usage: Any) -> dict[str, int | None]:
159
+ """Normalize any usage block DSPy can hand us into cmpnd's span counts.
160
+
161
+ The one place the SDK reads a provider's usage, because there are three
162
+ shapes and reading them at each call site is what recorded zero prompt and
163
+ completion tokens on every Responses-API span, next to a nonzero total:
164
+
165
+ * **chat raw** — ``prompt_tokens`` / ``completion_tokens`` /
166
+ ``total_tokens``, ``prompt_tokens_details.cached_tokens``,
167
+ ``completion_tokens_details.reasoning_tokens``.
168
+ * **Responses raw** — ``input_tokens`` / ``output_tokens`` /
169
+ ``total_tokens``, ``input_tokens_details.cached_tokens``,
170
+ ``output_tokens_details.reasoning_tokens``. A stock ``dspy.LM`` routes
171
+ through the legacy contract, whose history entry stores the provider's
172
+ block verbatim, so this shape arrives unnormalized.
173
+ * **DSPy typed ``LMUsage``** — fills both name pairs and carries top-level
174
+ ``reasoning_tokens`` / ``cache_read_tokens`` / ``cache_write_tokens``.
175
+
176
+ Anthropic's native ``cache_read_input_tokens`` and
177
+ ``cache_creation_input_tokens`` ride along on the raw paths.
178
+
179
+ ``prompt_tokens``, ``completion_tokens``, ``total_tokens`` and
180
+ ``cached_prompt_tokens`` come back as counts (0 when unreported — the
181
+ columns behind them are ``NOT NULL``). ``reasoning_tokens`` and
182
+ ``cache_write_tokens`` come back as ``None`` when the provider said nothing.
183
+ """
184
+ prompt = _reported_count(
185
+ _usage_field(usage, "prompt_tokens"),
186
+ _usage_field(usage, "input_tokens"),
187
+ )
188
+ completion = _reported_count(
189
+ _usage_field(usage, "completion_tokens"),
190
+ _usage_field(usage, "output_tokens"),
191
+ )
192
+ total = _reported_count(_usage_field(usage, "total_tokens"))
193
+ if total is None and (prompt is not None or completion is not None):
194
+ # Derived only where the provider omitted it — a reported total is
195
+ # evidence and stays as sent, even when the split disagrees with it.
196
+ total = (prompt or 0) + (completion or 0)
197
+
198
+ cached = _reported_count(
199
+ _usage_detail(usage, "prompt_tokens_details", "cached_tokens"),
200
+ _usage_detail(usage, "input_tokens_details", "cached_tokens"),
201
+ _usage_field(usage, "cache_read_tokens"),
202
+ _usage_field(usage, "cache_read_input_tokens"),
203
+ )
204
+ reasoning = _reported_count(
205
+ _usage_detail(usage, "completion_tokens_details", "reasoning_tokens"),
206
+ _usage_detail(usage, "output_tokens_details", "reasoning_tokens"),
207
+ _usage_field(usage, "reasoning_tokens"),
208
+ )
209
+ cache_write = _reported_count(
210
+ _usage_field(usage, "cache_write_tokens"),
211
+ _usage_field(usage, "cache_creation_input_tokens"),
212
+ )
213
+
214
+ return {
215
+ "prompt_tokens": prompt or 0,
216
+ "completion_tokens": completion or 0,
217
+ "total_tokens": total or 0,
218
+ # A subset of prompt_tokens: every dialect above counts its cache reads
219
+ # inside the prompt total, so the subset invariant holds by construction.
220
+ "cached_prompt_tokens": cached or 0,
221
+ "reasoning_tokens": reasoning,
222
+ "cache_write_tokens": cache_write,
223
+ }
224
+
225
+
226
+ def _finish_reason(response: Any) -> str | None:
227
+ """Read why the model stopped, from whichever response shape answered.
228
+
229
+ Chat puts it on a choice. DSPy's typed ``LMResponse`` puts it on an output.
230
+ The Responses API has neither: it reports a ``status`` plus, when the answer
231
+ was cut short, an ``incomplete_details.reason``. Those map onto the chat
232
+ vocabulary, which is what serve's truncation warning reads — a Responses
233
+ ``max_output_tokens`` and a chat ``length`` are the same fact and must land
234
+ on the same word.
235
+
236
+ Returns ``None`` when the response reports nothing about it.
237
+ """
238
+ if response is None:
239
+ return None
240
+
241
+ choices = getattr(response, "choices", None)
242
+ if choices:
243
+ reason = getattr(choices[0], "finish_reason", None)
244
+ if reason:
245
+ return str(reason)
246
+
247
+ outputs = getattr(response, "outputs", None)
248
+ if isinstance(outputs, list) and outputs:
249
+ reason = getattr(outputs[0], "finish_reason", None)
250
+ if reason:
251
+ return str(reason)
252
+
253
+ incomplete = getattr(response, "incomplete_details", None)
254
+ reason = incomplete.get("reason") if isinstance(incomplete, dict) else getattr(incomplete, "reason", None)
255
+ if reason:
256
+ return "length" if reason == "max_output_tokens" else str(reason)
257
+
258
+ status = getattr(response, "status", None)
259
+ if not isinstance(status, str) or not status:
260
+ return None
261
+ if status != "completed":
262
+ # "failed", "in_progress" — recorded as reported rather than guessed at.
263
+ return status
264
+ items = getattr(response, "output", None) or []
265
+ for item in items:
266
+ kind = item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
267
+ if kind == "function_call":
268
+ return "tool_calls"
269
+ return "stop"
119
270
 
120
271
 
121
272
  class _CallbackBase:
@@ -512,7 +663,6 @@ class _ModuleMixin(_CallbackBase):
512
663
 
513
664
  exp = exporter.get_exporter()
514
665
  if exp:
515
- exp.export_trace_stats(trace) # Stats tier (lightweight, never dropped)
516
666
  if trace.streaming:
517
667
  exp.export_trace_finalize(trace) # Finalize the streaming trace
518
668
  else:
@@ -734,15 +884,30 @@ class _ModuleMixin(_CallbackBase):
734
884
  return trace
735
885
 
736
886
  def _extract_usage_from_prediction(self, span: models.SpanBuilder, outputs: Any, dspy: Any) -> None:
737
- """Extract token usage from a DSPy Prediction."""
738
- if isinstance(outputs, dspy.Prediction):
739
- usage_by_model = outputs.get_lm_usage() if hasattr(outputs, "get_lm_usage") else None
740
- if usage_by_model:
741
- for model_usage in usage_by_model.values():
742
- span.prompt_tokens += model_usage.get("prompt_tokens", 0)
743
- span.completion_tokens += model_usage.get("completion_tokens", 0)
744
- span.total_tokens += model_usage.get("total_tokens", 0)
745
- span.cached_prompt_tokens += _cached_prompt_tokens(model_usage)
887
+ """Sum the LM usage a DSPy Prediction carries onto its module span.
888
+
889
+ This is what fills the trace row's own token totals, so a shape read
890
+ wrongly here reports a trace as having spent no tokens even when the LM
891
+ spans below it carry the right counts.
892
+ """
893
+ if not isinstance(outputs, dspy.Prediction):
894
+ return
895
+ usage_by_model = outputs.get_lm_usage() if hasattr(outputs, "get_lm_usage") else None
896
+ if not usage_by_model:
897
+ return
898
+ for model_usage in usage_by_model.values():
899
+ counts = _normalize_usage(model_usage)
900
+ span.prompt_tokens += counts["prompt_tokens"] or 0
901
+ span.completion_tokens += counts["completion_tokens"] or 0
902
+ span.total_tokens += counts["total_tokens"] or 0
903
+ span.cached_prompt_tokens += counts["cached_prompt_tokens"] or 0
904
+ # Summed only over the models that reported one, so a span whose
905
+ # models all stayed silent keeps the field absent rather than 0.
906
+ for name in ("reasoning_tokens", "cache_write_tokens"):
907
+ reported = counts[name]
908
+ if reported is None:
909
+ continue
910
+ setattr(span, name, (getattr(span, name) or 0) + reported)
746
911
 
747
912
 
748
913
  class _LMMixin(_CallbackBase):
@@ -841,11 +1006,13 @@ class _LMMixin(_CallbackBase):
841
1006
  usage = outputs.get("usage", {})
842
1007
 
843
1008
  if usage:
844
- span.prompt_tokens = usage.get("prompt_tokens", 0)
845
- span.completion_tokens = usage.get("completion_tokens", 0)
846
- span.total_tokens = usage.get("total_tokens", 0)
847
- span.cached_prompt_tokens = _cached_prompt_tokens(usage)
848
- span.cost = helpers.compute_cost(span.model_name or "", span.prompt_tokens, span.completion_tokens)
1009
+ counts = _normalize_usage(usage)
1010
+ span.prompt_tokens = counts["prompt_tokens"] or 0
1011
+ span.completion_tokens = counts["completion_tokens"] or 0
1012
+ span.total_tokens = counts["total_tokens"] or 0
1013
+ span.cached_prompt_tokens = counts["cached_prompt_tokens"] or 0
1014
+ span.reasoning_tokens = counts["reasoning_tokens"]
1015
+ span.cache_write_tokens = counts["cache_write_tokens"]
849
1016
  logger.debug(f"LM usage captured: {span.total_tokens} tokens")
850
1017
  else:
851
1018
  logger.debug(f"NO usage data for {span.model_name} (call_id={call_id})")
@@ -854,11 +1021,9 @@ class _LMMixin(_CallbackBase):
854
1021
  if instance and hasattr(instance, "history") and instance.history:
855
1022
  try:
856
1023
  latest_entry = instance.history[-1]
857
- response = latest_entry.get("response")
858
- if response and hasattr(response, "choices") and response.choices:
859
- finish_reason = getattr(response.choices[0], "finish_reason", None)
860
- if finish_reason:
861
- span.set_attribute("lm.finish_reason", finish_reason)
1024
+ finish_reason = _finish_reason(latest_entry.get("response"))
1025
+ if finish_reason:
1026
+ span.set_attribute("lm.finish_reason", finish_reason)
862
1027
  except (IndexError, AttributeError, TypeError):
863
1028
  pass
864
1029
 
@@ -906,6 +1071,13 @@ class _LMMixin(_CallbackBase):
906
1071
  separate judge LM), run during candidate evaluation after the program
907
1072
  trace has already closed.
908
1073
 
1074
+ Only a metric that calls the LM **directly** arrives here. A metric that
1075
+ grades through a dspy module opens a trace of its own at that module's
1076
+ boundary, so its LM call has a trace in scope and never reaches this
1077
+ function — it is an ordinary trace, linked to the eval run and named
1078
+ after the grading module. So the two ways of writing a judge take
1079
+ completely different paths through this callback.
1080
+
909
1081
  We distinguish them by `_active_eval_run` — set only while DSPy Evaluate
910
1082
  is running — instead of assuming every untraced call is reflection (that
911
1083
  would mislabel judge calls). The trace is named with the umbrella "GEPA";
@@ -949,7 +1121,6 @@ class _LMMixin(_CallbackBase):
949
1121
 
950
1122
  exp = exporter.get_exporter()
951
1123
  if exp:
952
- exp.export_trace_stats(trace) # stats tier (never dropped)
953
1124
  trace._batch_spans = [span]
954
1125
  exp.export_trace(trace)
955
1126
 
@@ -260,13 +260,27 @@ class EvalHandler:
260
260
  elif isinstance(outputs, tuple) and len(outputs) >= 2:
261
261
  results = outputs[1] if isinstance(outputs[1], list) else []
262
262
 
263
- # Get collected traces for this eval run (in execution order)
263
+ # Every trace this eval run collected, in execution order. The order
264
+ # matters only for _last_eval_trace_ids below — examples are matched
265
+ # by identity, never by position (see traces_by_id).
264
266
  eval_run_id = str(eval_run.eval_run_id)
265
267
  eval_traces = self._eval_traces.get(eval_run_id, [])
266
- # Sort traces by start_time to ensure correct ordering
267
268
  _min_time = datetime.min.replace(tzinfo=timezone.utc)
268
269
  eval_traces.sort(key=lambda t: t.start_time or _min_time)
269
270
 
271
+ # A metric that asks a model to grade the answer makes its call
272
+ # after the program's own trace has closed, so the grader opens a
273
+ # trace of its own and this list holds roughly twice as many traces
274
+ # as there are results. Handing result N the Nth trace therefore
275
+ # gives an example whichever run happened to sort into its slot —
276
+ # the grader's tokens, the grader's cost and a trace link pointing
277
+ # at a grading call. The callback stamps each returned Prediction
278
+ # with the trace id of the run that produced it, so that stamp —
279
+ # not a position — is what a result is matched on.
280
+ traces_by_id = {str(t.trace_id): t for t in eval_traces}
281
+ # Imported here, not at module scope: callback imports eval_handler.
282
+ from . import callback as callback_module
283
+
270
284
  logger.debug(f"Collected {len(eval_traces)} traces for eval run")
271
285
  for i, t in enumerate(eval_traces[:3]): # Show first 3
272
286
  logger.debug(f" Trace {i}: tokens={t.total_tokens}, cost={t.total_cost}")
@@ -322,9 +336,13 @@ class EvalHandler:
322
336
  # Compute example hash for comparability
323
337
  example_builder.example_hash = identity.compute_example_hash(example_builder.inputs)
324
338
 
325
- # Match with trace to get metrics (by index)
326
- if idx < len(eval_traces):
327
- trace = eval_traces[idx]
339
+ # Match this result to the run that produced it, by the
340
+ # trace id the callback stamped on the prediction. No
341
+ # positional fallback: a result carrying no stamp is left
342
+ # unlinked, because a wrong trace, a wrong token count and a
343
+ # wrong cost are worse than none.
344
+ trace = traces_by_id.get(callback_module.trace_id(prediction) or "")
345
+ if trace is not None:
328
346
  example_builder.trace_id = trace.trace_id
329
347
 
330
348
  # Extract duration from trace
@@ -335,6 +353,8 @@ class EvalHandler:
335
353
  # Extract tokens and cost from trace
336
354
  example_builder.total_tokens = trace.total_tokens
337
355
  example_builder.cost = trace.total_cost
356
+ else:
357
+ logger.debug(f"Eval example {idx} carries no trace stamp; left unattributed")
338
358
 
339
359
  eval_run.add_example_result(example_builder)
340
360
 
@@ -126,31 +126,3 @@ def infer_provider(instance: Any) -> str:
126
126
  return "together"
127
127
  else:
128
128
  return "unknown"
129
-
130
-
131
- # Simple cost table for common models ($/1M tokens)
132
- # (input_cost_per_1m, output_cost_per_1m)
133
- _MODEL_COSTS: dict[str, tuple[float, float]] = {
134
- "gpt-4o": (2.50, 10.00),
135
- "gpt-4o-mini": (0.15, 0.60),
136
- "gpt-4-turbo": (10.00, 30.00),
137
- "gpt-4": (30.00, 60.00),
138
- "gpt-3.5-turbo": (0.50, 1.50),
139
- "claude-3-5-sonnet-20241022": (3.00, 15.00),
140
- "claude-sonnet-4-20250514": (3.00, 15.00),
141
- "claude-3-5-haiku-20241022": (0.80, 4.00),
142
- "claude-haiku-4-5-20251001": (0.80, 4.00),
143
- "claude-3-opus-20240229": (15.00, 75.00),
144
- "claude-opus-4-20250514": (15.00, 75.00),
145
- }
146
-
147
-
148
- def compute_cost(model_name: str, prompt_tokens: int, completion_tokens: int) -> float:
149
- """Compute cost from model name and token counts."""
150
- if not model_name or (prompt_tokens == 0 and completion_tokens == 0):
151
- return 0.0
152
- costs = _MODEL_COSTS.get(model_name)
153
- if not costs:
154
- return 0.0
155
- input_cost, output_cost = costs
156
- return (prompt_tokens * input_cost / 1_000_000) + (completion_tokens * output_cost / 1_000_000)
@@ -461,6 +461,18 @@ class SpanBuilder:
461
461
  # the cached reads — so cached_prompt_tokens <= prompt_tokens holds by
462
462
  # construction. 0 means no cache reads (or the provider reported none).
463
463
  cached_prompt_tokens: int = 0
464
+ # Reasoning (thinking) tokens the provider billed inside completion_tokens,
465
+ # and prompt tokens it charged for WRITING into its prompt cache. Both are
466
+ # None when the provider reported nothing about them — a call that said
467
+ # nothing and a call that reported none are different facts, and pricing
468
+ # treats them differently, so absent must not be flattened into zero.
469
+ # Normalized by callback._normalize_usage.
470
+ reasoning_tokens: int | None = None
471
+ cache_write_tokens: int | None = None
472
+ # Left unset by the SDK. A dollar figure is a derivation from these counts
473
+ # and a dated rate catalog, and only serve holds the catalog; the SDK's job
474
+ # is to report what the provider said. serve still reads the wire field, so
475
+ # it stays on the model for a producer that does know a price.
464
476
  cost: float = 0.0
465
477
 
466
478
  # DSPy-specific: Demo info
@@ -568,6 +580,8 @@ class SpanBuilder:
568
580
  "completion_tokens": self.completion_tokens,
569
581
  "total_tokens": self.total_tokens,
570
582
  "cached_prompt_tokens": self.cached_prompt_tokens,
583
+ "reasoning_tokens": self.reasoning_tokens,
584
+ "cache_write_tokens": self.cache_write_tokens,
571
585
  "cost": self.cost,
572
586
  "demos": _safe_serialize(self.demos),
573
587
  "demo_count": self.demo_count if self.demo_count else len(self.demos),
@@ -728,26 +742,6 @@ class TraceBuilder:
728
742
  "project_tags": self.project_tags if self.project_tags else [],
729
743
  }
730
744
 
731
- def to_stats_dict(self) -> dict[str, Any]:
732
- """Lightweight stats payload (~200 bytes). Never dropped."""
733
- return {
734
- "trace_id": str(self.trace_id),
735
- "timestamp": self.start_time.isoformat() if self.start_time else None,
736
- "program_signature_hash": identity.format_signature_hash(self.program_signature_hash),
737
- "program_name": self.program_name or "",
738
- "status": self.status.value,
739
- "total_tokens": self.total_tokens,
740
- "prompt_tokens": self.prompt_tokens,
741
- "completion_tokens": self.completion_tokens,
742
- "total_cost": self.total_cost,
743
- "model_name": "",
744
- "eval_run_id": str(self.eval_run_id) if self.eval_run_id else None,
745
- "optimization_run_id": (str(self.optimization_run_id) if self.optimization_run_id else None),
746
- "span_count": self.span_count,
747
- "project": self.project_tags[0] if self.project_tags else None,
748
- "project_tags": self.project_tags if self.project_tags else [],
749
- }
750
-
751
745
 
752
746
  class EvalRunStatus(str, Enum):
753
747
  """Evaluation run status."""
@@ -47,7 +47,6 @@ cmpnd/
47
47
  → Aggregate tokens/cost across spans
48
48
  → If streaming (RLM): export_trace_start + export_span per child + export_trace_finalize
49
49
  → If batching: export_trace with all spans attached
50
- → Always: export_trace_stats (lightweight, never dropped)
51
50
 
52
51
  4. Background exporter thread
53
52
  → Collects from queue with timeout
@@ -180,7 +180,7 @@ Override any of these by passing kwargs to `cmpnd.configure(...)`.
180
180
 
181
181
  - `start()` at `exporter_http.py:131` creates the worker thread and the `ThreadPoolExecutor`, then registers `shutdown()` via `threading._register_atexit` (or `atexit.register` as a fallback). The registration ordering is deliberate — there's a comment at `exporter_http.py:139` explaining why it has to go after the executor init.
182
182
  - `_export_loop` at `exporter_http.py:355` pulls from the internal `queue.Queue` with a 100ms timeout, appends to a local batch, and calls `_flush_batch` when `batch_size` or `flush_interval_seconds` trips.
183
- - `_flush_batch` at `exporter_http.py:443` sorts items by type (`trace`, `span`, `trace_start`, `trace_finalize`, `trace_stats`, `eval_run`, `optimization_run`, various optimization progress records), buffers loose spans keyed by `trace_id` so they can be attached to their parent trace when it arrives, and submits each type's `_send_*` call to the thread pool in parallel.
183
+ - `_flush_batch` at `exporter_http.py:443` sorts items by type (`trace`, `span`, `trace_start`, `trace_finalize`, `eval_run`, `optimization_run`, various optimization progress records), buffers loose spans keyed by `trace_id` so they can be attached to their parent trace when it arrives, and submits each type's `_send_*` call to the thread pool in parallel.
184
184
  - `shutdown()` at `exporter_http.py:163` sets the shutdown event, joins the worker thread, drains any leftover queue items, then shuts down the executor and HTTP client. It is safe to call multiple times.
185
185
 
186
186
  ### Delivery errors, dropping, and transient retry
@@ -400,6 +400,8 @@ The trace payload is built in `python-sdk/cmpnd/models.py` by `TraceBuilder.to_a
400
400
  "prompt_tokens": 137,
401
401
  "completion_tokens": 42,
402
402
  "cached_prompt_tokens": 96, # provider prompt-cache reads, subset of prompt_tokens
403
+ "reasoning_tokens": 12, # subset of completion_tokens; None when unreported
404
+ "cache_write_tokens": None, # prompt-cache writes; None when unreported
403
405
  "inputs": { ... },
404
406
  "outputs": { ... },
405
407
  # ...plus status, timings, attributes, events
@@ -61,9 +61,6 @@ class TraceBody:
61
61
  def to_finalize_dict(self) -> dict[str, Any]:
62
62
  return dict(self.payload)
63
63
 
64
- def to_stats_dict(self) -> dict[str, Any]:
65
- return dict(self.payload)
66
-
67
64
 
68
65
  @dataclass(frozen=True)
69
66
  class SpanBody:
@@ -180,8 +180,7 @@ class ExportItem:
180
180
  """Item in the export queue."""
181
181
 
182
182
  # item_type: "trace", "span", "eval_run", "optimization_complete",
183
- # "optimization_progress", "trace_start", "trace_finalize",
184
- # or "trace_stats"
183
+ # "optimization_progress", "trace_start", or "trace_finalize"
185
184
  item_type: str
186
185
  # Tag-driven dispatch (item_type → method on data). pyright can't
187
186
  # narrow the union across the if/elif chain in _flush_batch, so each
@@ -588,10 +587,6 @@ class BatchExporter:
588
587
  """Queue a trace finalization for export (non-blocking)."""
589
588
  self._enqueue(ExportItem("trace_finalize", trace, time.time()))
590
589
 
591
- def export_trace_stats(self, trace: TraceRecord) -> None:
592
- """Queue trace stats for export (non-blocking)."""
593
- self._enqueue(ExportItem("trace_stats", trace, time.time()))
594
-
595
590
  def export_optimization_start(self, optimization_run: OptimizationRunRecord) -> str:
596
591
  """Create the optimization record server-side and return the
597
592
  server-assigned ``optimization_run_id``.
@@ -1137,7 +1132,6 @@ class BatchExporter:
1137
1132
  # --- Phase 1: Organize items by type (sequential) ---
1138
1133
 
1139
1134
  traces_to_send: list[dict[str, Any]] = []
1140
- stats_to_send: list[dict[str, Any]] = []
1141
1135
  eval_runs_to_send: list[dict[str, Any]] = []
1142
1136
  optimization_complete_to_send: list[dict[str, Any]] = []
1143
1137
  optimization_progress_to_send: list[tuple[str, dict[str, Any], OptimizationRunRecord]] = []
@@ -1149,8 +1143,6 @@ class BatchExporter:
1149
1143
  for item in batch:
1150
1144
  if item.item_type == "trace":
1151
1145
  traces_to_send.append(cast("TraceRecord", item.data).to_api_dict())
1152
- elif item.item_type == "trace_stats":
1153
- stats_to_send.append(cast("TraceRecord", item.data).to_stats_dict())
1154
1146
  elif item.item_type == "eval_run":
1155
1147
  eval_runs_to_send.append(cast("EvalRunRecord", item.data).to_api_dict())
1156
1148
  elif item.item_type == "optimization_complete":
@@ -1237,8 +1229,6 @@ class BatchExporter:
1237
1229
  futures.append(executor.submit(self._send_spans_for_trace, trace_id, span_list))
1238
1230
  if trace_finalizes:
1239
1231
  futures.append(executor.submit(self._send_trace_finalizes, trace_finalizes))
1240
- if stats_to_send:
1241
- futures.append(executor.submit(self._send_stats, stats_to_send))
1242
1232
  if eval_runs_to_send:
1243
1233
  futures.append(executor.submit(self._send_eval_runs, eval_runs_to_send))
1244
1234
  if optimization_progress_to_send:
@@ -1368,19 +1358,6 @@ class BatchExporter:
1368
1358
  with self._stats_lock:
1369
1359
  self._error_count += 1
1370
1360
 
1371
- def _send_stats(self, stats_to_send: list[dict[str, Any]]) -> None:
1372
- client = self._require_client()
1373
- try:
1374
- response = self._send_json_retrying(client, "POST", "/api/v1/traces/stats", stats_to_send)
1375
- response.raise_for_status()
1376
- with self._stats_lock:
1377
- self._exported_count += len(stats_to_send)
1378
- logger.debug(f"Exported {len(stats_to_send)} trace stats")
1379
- except Exception as e:
1380
- logger.error(f"Failed to export trace stats: {e}")
1381
- with self._stats_lock:
1382
- self._error_count += len(stats_to_send)
1383
-
1384
1361
  def _send_eval_runs(self, eval_runs_to_send: list[dict[str, Any]]) -> None:
1385
1362
  # Completion is a PATCH to the running record created by the
1386
1363
  # synchronous export_eval_start, keyed by the server-assigned id.
@@ -49,14 +49,13 @@ class SpanRecord(Protocol):
49
49
 
50
50
 
51
51
  class TraceRecord(Protocol):
52
- """A trace, in each of the four shapes the contract's routes take."""
52
+ """A trace, in each of the three shapes the contract's routes take."""
53
53
 
54
54
  @property
55
55
  def trace_id(self) -> Any: ...
56
56
  def to_api_dict(self) -> dict[str, Any]: ...
57
57
  def to_start_dict(self) -> dict[str, Any]: ...
58
58
  def to_finalize_dict(self) -> dict[str, Any]: ...
59
- def to_stats_dict(self) -> dict[str, Any]: ...
60
59
 
61
60
 
62
61
  class EvalRunRecord(Protocol):
@@ -5,7 +5,7 @@ name = "cmpnd"
5
5
  # it's new) — no git tags. Bump with `uv version --bump {minor,patch}` or by
6
6
  # hand. Runtime `cmpnd.__version__` reads installed metadata, so it tracks
7
7
  # this automatically. See docs/plans/2026-07-02-sdk-pypi-tag-free-publish.md.
8
- version = "0.20.0"
8
+ version = "0.22.0"
9
9
  description = "DSPy observability and deployment SDK for cmpnd"
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.10"
@@ -75,8 +75,6 @@ def _kind_for(method: str, path: str, payload) -> tuple[str, object] | None:
75
75
  return ("traces_batch", [payload])
76
76
  if _SPANS_PATH.match(path):
77
77
  return ("spans", payload)
78
- if path == "/api/v1/traces/stats":
79
- return ("stats", payload)
80
78
  elif method == "PUT" and _OPT_COMPLETE_PATH.match(path):
81
79
  if isinstance(payload, dict) and payload.get("status") == "running":
82
80
  return None
@@ -815,25 +815,21 @@ def main() -> int:
815
815
  _post(endpoint, f"/api/v1/optimizations/{new_op}", opt_payload, api_key, method="PUT")
816
816
  log.info("seeded optimization run %s", new_op)
817
817
 
818
- # 2) Replay its traces / spans / stats, linked to the new run.
819
- n_traces = n_spans = n_stats = 0
818
+ # 2) Replay its traces / spans, linked to the new run.
819
+ n_traces = n_spans = 0
820
820
  for p in parts:
821
821
  if p["kind"] == "traces_batch":
822
822
  payload = [_transform(item, uuid_map, delta, new_op, _TAGS_COMPLETE) for item in p["payload"]]
823
823
  _stamp_deployments(payload, dep_for)
824
824
  _post(endpoint, "/api/v1/traces/batch", payload, api_key)
825
825
  n_traces += len(payload)
826
- elif p["kind"] == "stats":
827
- payload = [_transform(item, uuid_map, delta, new_op, _TAGS_COMPLETE) for item in p["payload"]]
828
- _post(endpoint, "/api/v1/traces/stats", payload, api_key)
829
- n_stats += len(payload)
830
826
  elif p["kind"] == "spans":
831
827
  old = _UUID_RE.search(p["path"])
832
828
  trace_uuid = uuid_map.get(old.group(0), old.group(0)) if old else ""
833
829
  payload = _transform(p["payload"], uuid_map, delta, new_op, _TAGS_COMPLETE)
834
830
  _post(endpoint, f"/api/v1/traces/{trace_uuid}/spans", payload, api_key)
835
831
  n_spans += 1
836
- log.info("seeded %d traces, %d span-batches, %d stat rows -> %s", n_traces, n_spans, n_stats, new_op)
832
+ log.info("seeded %d traces, %d span-batches -> %s", n_traces, n_spans, new_op)
837
833
 
838
834
  # 2b) Pin the completed committee program for the demo admin (CMP-634) so
839
835
  # the sidebar shows a pinned program. Runs now that the trace ingest has
@@ -872,11 +868,6 @@ def main() -> int:
872
868
  payload = [_transform(item, umap, delta, running_op, _TAGS_RUNNING) for item in kept]
873
869
  _post(endpoint, "/api/v1/traces/batch", payload, api_key)
874
870
  n_it += len(payload)
875
- elif p["kind"] == "stats":
876
- kept = [s for s in p["payload"] if s.get("trace_id") in keep]
877
- if kept:
878
- payload = [_transform(item, umap, delta, running_op, _TAGS_RUNNING) for item in kept]
879
- _post(endpoint, "/api/v1/traces/stats", payload, api_key)
880
871
  elif p["kind"] == "spans":
881
872
  old = _UUID_RE.search(p["path"])
882
873
  if old and old.group(0) in keep: