ebaml 0.7.0__tar.gz → 0.7.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (203) hide show
  1. {ebaml-0.7.0 → ebaml-0.7.2}/PKG-INFO +2 -2
  2. {ebaml-0.7.0 → ebaml-0.7.2}/README.md +1 -1
  3. {ebaml-0.7.0 → ebaml-0.7.2}/pyproject.toml +1 -1
  4. {ebaml-0.7.0 → ebaml-0.7.2}/pyproject.toml.orig +1 -1
  5. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/.baml-generator-output.json +2 -2
  6. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/_baml_sources.py +1 -1
  7. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/_inlinedbaml.py +1 -1
  8. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/__init__.py +4 -2
  9. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/core/custom_types.py +4 -0
  10. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/core/exceptions.py +4 -0
  11. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/edit_records.py +4 -4
  12. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/history.py +17 -17
  13. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/descriptors.py +3 -2
  14. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/requests.py +1 -1
  15. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/audience.py +35 -35
  16. ebaml-0.7.2/src/ebaml/local/completions.py +101 -0
  17. ebaml-0.7.2/src/ebaml/local/counterfactuals.py +281 -0
  18. ebaml-0.7.2/src/ebaml/local/fact_features.py +329 -0
  19. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/fact_text.py +6 -6
  20. ebaml-0.7.2/src/ebaml/local/fact_types.py +230 -0
  21. ebaml-0.7.2/src/ebaml/local/facts.py +179 -0
  22. ebaml-0.7.2/src/ebaml/local/sufficient_reasons.py +207 -0
  23. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/chat.py +2 -2
  24. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/explain_tools.py +2 -1
  25. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/model_store.py +110 -4
  26. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/server.py +32 -5
  27. ebaml-0.7.0/src/ebaml/local/facts.py +0 -1274
  28. {ebaml-0.7.0 → ebaml-0.7.2}/LICENSE +0 -0
  29. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/.baml-generator-output-root +0 -0
  30. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/__init__.py +0 -0
  31. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/__init__.pyi +0 -0
  32. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/_function_registry.py +0 -0
  33. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/_invocation_types.py +0 -0
  34. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/_invocation_types.pyi +0 -0
  35. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/_typemap.py +0 -0
  36. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/__init__.py +0 -0
  37. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/__init__.pyi +0 -0
  38. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/clients/__init__.py +0 -0
  39. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/clients/__init__.pyi +0 -0
  40. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/content/__init__.py +0 -0
  41. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/content/__init__.pyi +0 -0
  42. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/errors/__init__.py +0 -0
  43. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/errors/__init__.pyi +0 -0
  44. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/events/__init__.py +0 -0
  45. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/events/__init__.pyi +0 -0
  46. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/internal/__init__.py +0 -0
  47. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/internal/__init__.pyi +0 -0
  48. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/mcp/__init__.py +0 -0
  49. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/mcp/__init__.pyi +0 -0
  50. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/stream/__init__.py +0 -0
  51. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/stream/__init__.pyi +0 -0
  52. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/tools/__init__.py +0 -0
  53. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/tools/__init__.pyi +0 -0
  54. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/wire/__init__.py +0 -0
  55. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/ai/wire/__init__.pyi +0 -0
  56. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/__init__.py +0 -0
  57. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/__init__.pyi +0 -0
  58. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/crypto/__init__.py +0 -0
  59. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/crypto/__init__.pyi +0 -0
  60. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/csv/__init__.py +0 -0
  61. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/csv/__init__.pyi +0 -0
  62. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/env/__init__.py +0 -0
  63. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/env/__init__.pyi +0 -0
  64. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/errors/__init__.py +0 -0
  65. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/errors/__init__.pyi +0 -0
  66. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/fs/__init__.py +0 -0
  67. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/fs/__init__.pyi +0 -0
  68. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/future/__init__.py +0 -0
  69. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/future/__init__.pyi +0 -0
  70. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/glob/__init__.py +0 -0
  71. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/glob/__init__.pyi +0 -0
  72. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/hash/__init__.py +0 -0
  73. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/hash/__init__.pyi +0 -0
  74. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/host/__init__.py +0 -0
  75. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/host/__init__.pyi +0 -0
  76. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/http/__init__.py +0 -0
  77. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/http/__init__.pyi +0 -0
  78. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/io/__init__.py +0 -0
  79. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/io/__init__.pyi +0 -0
  80. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/iter/__init__.py +0 -0
  81. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/iter/__init__.pyi +0 -0
  82. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/json/__init__.py +0 -0
  83. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/json/__init__.pyi +0 -0
  84. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/media/__init__.py +0 -0
  85. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/media/__init__.pyi +0 -0
  86. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/net/__init__.py +0 -0
  87. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/net/__init__.pyi +0 -0
  88. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/ops/__init__.py +0 -0
  89. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/ops/__init__.pyi +0 -0
  90. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/panics/__init__.py +0 -0
  91. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/panics/__init__.pyi +0 -0
  92. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/random/__init__.py +0 -0
  93. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/random/__init__.pyi +0 -0
  94. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/regex/__init__.py +0 -0
  95. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/regex/__init__.pyi +0 -0
  96. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/sap/__init__.py +0 -0
  97. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/sap/__init__.pyi +0 -0
  98. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/spawn/__init__.py +0 -0
  99. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/spawn/__init__.pyi +0 -0
  100. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/sys/__init__.py +0 -0
  101. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/sys/__init__.pyi +0 -0
  102. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/time/__init__.py +0 -0
  103. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/time/__init__.pyi +0 -0
  104. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/toml/__init__.py +0 -0
  105. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/toml/__init__.pyi +0 -0
  106. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/ws/__init__.py +0 -0
  107. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/ws/__init__.pyi +0 -0
  108. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/yaml/__init__.py +0 -0
  109. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/baml/yaml/__init__.pyi +0 -0
  110. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/experimental/__init__.py +0 -0
  111. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/experimental/__init__.pyi +0 -0
  112. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/py.typed +0 -0
  113. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/__init__.py +0 -0
  114. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/__init__.pyi +0 -0
  115. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/class_/__init__.py +0 -0
  116. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/class_/__init__.pyi +0 -0
  117. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/enum/__init__.py +0 -0
  118. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/enum/__init__.pyi +0 -0
  119. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/errors/__init__.py +0 -0
  120. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/errors/__init__.pyi +0 -0
  121. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/function/__init__.py +0 -0
  122. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/function/__init__.pyi +0 -0
  123. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/interface/__init__.py +0 -0
  124. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/reflect/interface/__init__.pyi +0 -0
  125. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/__init__.py +0 -0
  126. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/__init__.pyi +0 -0
  127. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/anthropic/__init__.py +0 -0
  128. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/anthropic/__init__.pyi +0 -0
  129. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/anthropic/internal/__init__.py +0 -0
  130. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/anthropic/internal/__init__.pyi +0 -0
  131. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/assert_/__init__.py +0 -0
  132. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/assert_/__init__.pyi +0 -0
  133. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/aws/__init__.py +0 -0
  134. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/aws/__init__.pyi +0 -0
  135. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/aws/internal/__init__.py +0 -0
  136. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/aws/internal/__init__.pyi +0 -0
  137. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/claude_code/__init__.py +0 -0
  138. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/claude_code/__init__.pyi +0 -0
  139. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/claude_code/internal/__init__.py +0 -0
  140. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/claude_code/internal/__init__.pyi +0 -0
  141. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/google/__init__.py +0 -0
  142. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/google/__init__.pyi +0 -0
  143. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/google/internal/__init__.py +0 -0
  144. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/google/internal/__init__.pyi +0 -0
  145. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/openai/__init__.py +0 -0
  146. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/openai/__init__.pyi +0 -0
  147. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/openai/internal/__init__.py +0 -0
  148. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/openai/internal/__init__.pyi +0 -0
  149. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/openai/internal/decisions/__init__.py +0 -0
  150. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/openai/internal/decisions/__init__.pyi +0 -0
  151. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/testing/__init__.py +0 -0
  152. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/testing/__init__.pyi +0 -0
  153. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/trace/__init__.py +0 -0
  154. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/trace/__init__.pyi +0 -0
  155. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/typesafeai/__init__.py +0 -0
  156. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/typesafeai/__init__.pyi +0 -0
  157. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/typesafeai/internal/__init__.py +0 -0
  158. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/typesafeai/internal/__init__.pyi +0 -0
  159. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/vercel/__init__.py +0 -0
  160. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/vercel/__init__.pyi +0 -0
  161. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/vercel/internal/__init__.py +0 -0
  162. {ebaml-0.7.0 → ebaml-0.7.2}/src/baml_sdk/vendor/vercel/internal/__init__.pyi +0 -0
  163. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/benchmark.py +0 -0
  164. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/core/__init__.py +0 -0
  165. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/core/beartype_hook.py +0 -0
  166. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/core/ebm_classes.py +0 -0
  167. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/__init__.py +0 -0
  168. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/edit_report.py +0 -0
  169. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/edit_session.py +0 -0
  170. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/edits.py +0 -0
  171. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/model_view.py +0 -0
  172. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/model_write.py +0 -0
  173. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/revision_ids.py +0 -0
  174. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/edit/selections.py +0 -0
  175. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/__init__.py +0 -0
  176. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/ebm_graph.py +0 -0
  177. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/graph_text.py +0 -0
  178. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/model_kinds.py +0 -0
  179. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/request_text.py +0 -0
  180. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/terms.py +0 -0
  181. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/token_budget.py +0 -0
  182. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/graphs/token_weights.py +0 -0
  183. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/__init__.py +0 -0
  184. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/options.py +0 -0
  185. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/progress_bars.py +0 -0
  186. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/provider_errors.py +0 -0
  187. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/retry.py +0 -0
  188. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/sdk_setup.py +0 -0
  189. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/token_learning.py +0 -0
  190. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/llm/token_weights.json +0 -0
  191. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/__init__.py +0 -0
  192. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/covering_backend.py +0 -0
  193. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/covering_greedy.py +0 -0
  194. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/covering_problem.py +0 -0
  195. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/covering_solvers.py +0 -0
  196. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/decisions.py +0 -0
  197. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/local_kinds.py +0 -0
  198. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/minimality.py +0 -0
  199. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/local/ortools_backend.py +0 -0
  200. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/__init__.py +0 -0
  201. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/llm_tools.py +0 -0
  202. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/mcp/read_tools.py +0 -0
  203. {ebaml-0.7.0 → ebaml-0.7.2}/src/ebaml/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ebaml
3
- Version: 0.7.0
3
+ Version: 0.7.2
4
4
  Summary: BAML for Explainable Boosting Machines: talk to your EBM.
5
5
  Keywords: baml,llm,explainable-ai,explainable-boosting-machine,interpretml
6
6
  Author: Johnny Godoy
@@ -420,7 +420,7 @@ keeps it up for the whole chat. It needs the `mcp` extra and the BAML toolchain.
420
420
  `OPENROUTER_API_KEY`.
421
421
 
422
422
  ```bash
423
- ebaml-chat --model model.skops --data train.csv
423
+ ebaml-chat --model model.skops --data train.parquet
424
424
  ```
425
425
 
426
426
  ```text
@@ -378,7 +378,7 @@ keeps it up for the whole chat. It needs the `mcp` extra and the BAML toolchain.
378
378
  `OPENROUTER_API_KEY`.
379
379
 
380
380
  ```bash
381
- ebaml-chat --model model.skops --data train.csv
381
+ ebaml-chat --model model.skops --data train.parquet
382
382
  ```
383
383
 
384
384
  ```text
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ebaml"
3
- version = "0.7.0"
3
+ version = "0.7.2"
4
4
  description = "BAML for Explainable Boosting Machines: talk to your EBM."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ebaml"
3
- version = "0.7.0"
3
+ version = "0.7.2"
4
4
  description = "BAML for Explainable Boosting Machines: talk to your EBM."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -15,7 +15,7 @@
15
15
  },
16
16
  {
17
17
  "relative_path": "_baml_sources.py",
18
- "sha256": "cc68b20b63714bf0466bf54414c983617b0a8e7d4977a7db874e8f92298c1747"
18
+ "sha256": "2251876bdba10e9088ebc255613dafc72e95ff6a2569ac7ede3774cbed4eae47"
19
19
  },
20
20
  {
21
21
  "relative_path": "_function_registry.py",
@@ -23,7 +23,7 @@
23
23
  },
24
24
  {
25
25
  "relative_path": "_inlinedbaml.py",
26
- "sha256": "7fcc2dcb4bba17adaaa286938b2615b3f176018c61cb3d8504585f94c6db23e6"
26
+ "sha256": "f82027079b77066ad7acc2b80a27d27fe21ce9de6d218f5b6334324a3138b80f"
27
27
  },
28
28
  {
29
29
  "relative_path": "_invocation_types.py",
@@ -23,7 +23,7 @@ from __future__ import annotations
23
23
 
24
24
  FILES: dict[str, str] = {
25
25
  "local_explanations.baml": "// Local explanations: Python computes the facts of one prediction deterministically (ebaml.local), BAML turns each\n// kind of fact into words. The three kinds are the explainability queries of Barceló et al., \"Model Interpretability\n// through the Lens of Computational Complexity\" (arXiv:2010.12265): minimum change required, sufficient reason and\n// count completions. The global explanation of the model (describe_ebm) is prepended to every prompt.\n\n// Facts of one prediction, one JSON text per kind of explanation.\nclass LocalFactTexts {\n prediction: string,\n counterfactual: string,\n sufficient_reason: string,\n completions: string,\n}\n\nclass LocalExplanationText {\n counterfactual: string,\n sufficient_reason: string,\n completions: string,\n usage: TokenUsage,\n}\n\nfunction local_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You explain individual predictions of a Generalized Additive Model (GAM). You receive an explanation of the whole model and facts about one prediction that were computed exactly from the model. State only effects, values and probabilities that appear in these facts, because the reader acts on your words and an invented number would mislead. A prediction is a class or a number. For a number, the facts name the decision as a question about the predicted value, such as \"above 10\", and a flip means that the value crosses the limit of the question. For a class, a flip means that another class becomes the predicted class, and \"after_the_changes\" names it. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// The guidance comes from the Audience of the Python caller (ebaml.local.audience), so a user can add an audience\n// without an edit here. It tells the model how to write for the reader, and why.\nfunction local_user_msg(\n global_explanation: string,\n prediction: string,\n question: string,\n facts: string,\n audience_guidance: string,\n dataset_description: string,\n language: string = \"English\",\n) -> string {\n let msg = `\n <global_explanation>\n ${global_explanation}\n </global_explanation>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + `\n <prediction>\n ${prediction}\n </prediction>\n\n <facts>\n ${facts}\n </facts>\n\n <audience>\n ${audience_guidance}\n </audience>\n\n <language>\n Write the whole answer in ${language}, because the reader reads ${language}. Keep every number exactly as the facts give it, and translate the names of inputs and categories only where the reader needs them.\n </language>\n\n <question>\n ${question}\n </question>\n `\n}\n\nfunction counterfactual_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, what would have to change for the prediction to be different.\n The facts list a set of input changes that flips the prediction (a minimum change required), and whether the flip survives the model's uncertainty.\n If \"is_smallest_set\" is true, this is exact, not a guess: no smaller set of changes gives the new prediction, because the model adds up one independent effect per input and the computation proves the size.\n If \"is_smallest_set\" is false, the facts ask for a chosen class, and the search ended at its time limit or was a fast search, so it did not prove the size. Describe the set as a set of changes that reaches that class, and keep its size out of the answer, because any word about the size reads as a claim that the facts do not make.\n If the facts give \"smallest_possible_number_of_changes\", no set reaches the new prediction with fewer changes than that number. When \"is_smallest_set\" is false, you may state the gap between \"number_of_changes\" and that number, because the gap tells this reader how far the set can be from the smallest one.\n The search considered the values that the model saw in training. If \"rare_values_left_out\" is true, say that only realistic alternative values were considered, because the search left out rare extreme values. If it is false, make no such statement, because the search left out no value.\n If \"flip_holds_even_if_the_model_is_somewhat_off\" is true, or \"flip_holds_at_confidence\" is above 0, the flip holds even if the effects of the changed features are at the unfavourable edge of their confidence interval, and you may say so. Limit that statement to the changed features, because the effects of the other inputs are taken as the model fitted them. If the key is false or 0, every effect is taken as fitted, so make no statement about the model's uncertainty, because the facts support none.\n Suggest only the changes that the facts list, because they are the only ones that the model supports.\n If \"flip_possible\" is false, say that the search found no change that flips the prediction among the values that it considered: within the realistic values when \"rare_values_left_out\" is true, and within the values seen in training otherwise. The search covered only those values, so the facts make no claim about other values. When the confidence key is true or above 0, the search asked for a flip that survives the uncertainty of the changed features, so a change that flips the prediction only with the effects as fitted is outside this result.\n `\n}\n\nfunction sufficient_reason_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, which inputs are enough to settle the prediction.\n The facts list a set of inputs (a sufficient reason) that fixes the prediction whatever the other inputs are. State clearly that the remaining inputs cannot change the outcome within the values that the guarantee covers, and name the inputs of \"features_not_needed\", because the reader wants to know which inputs do not matter.\n If \"is_smallest_set\" is true, this is exact: no smaller set of inputs guarantees the outcome, because the model adds up one independent effect per input and the computation proves the size.\n If \"is_smallest_set\" is false, the search ended at its time limit or was a fast search, so it did not prove the size. Call the set a set of inputs that is enough, and keep its size out of the answer, because any word about the size reads as a claim that the facts do not make.\n If the facts give \"smallest_possible_number_of_features\", no set with fewer inputs than that number guarantees the outcome. When \"is_smallest_set\" is false, you may state the gap between the number of listed inputs and that number, because the gap tells this reader how far the set can be from the smallest one.\n The guarantee covers the values of the other inputs that the model saw in training. If \"rare_values_left_out\" is true, say that it covers every realistic value of the other inputs, because rare extreme values are left out. If it is false, it covers every value seen in training, so make no statement about rare values.\n If \"guarantee_holds_even_if_the_model_is_somewhat_off\" is true, or \"guarantee_holds_at_confidence\" is above 0, the guarantee holds even when the model's effects are at the unfavourable edge of their confidence interval, and you may say so. If the key is false or 0, every effect is taken as the model fitted it, so make no statement about the model's uncertainty, because the facts support none.\n Present it as a statement about this model, not about the real world.\n If there is no such set, say that the search found no set of inputs that settles the prediction under these conditions, because that result does not show that each single input is needed.\n `\n}\n\nfunction completions_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, how certain the outcome is as the inputs become known.\n The facts give the share of people who would get the same prediction when only the first inputs are known and the rest are unknown (count completions).\n Describe how that share changes from step to step, and say so when it falls, because learning more inputs can lower the share and a sample adds noise. Name the inputs that make the biggest difference.\n Unlike the other two explanations, these shares are estimates from a random sample that treats the unknown inputs as independent and distributed as in the training data. Describe them as approximate, because they are not guarantees.\n The facts give the size of the sample, as the word \"sample_size\" or as \"number_of_samples\". When the word is \"small\", or the number is below 1000, say that the shares are rough, because a small sample adds much noise.\n `\n}\n\nfunction LocalCounterfactual(\n global_explanation: string,\n facts: LocalFactTexts,\n audience_guidance: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n language: string = \"English\",\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, counterfactual_question(num_sentences), facts.counterfactual, audience_guidance, opts.dataset_description, language = language)}\n `\n}\n\nfunction LocalSufficientReason(\n global_explanation: string,\n facts: LocalFactTexts,\n audience_guidance: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n language: string = \"English\",\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, sufficient_reason_question(num_sentences), facts.sufficient_reason, audience_guidance, opts.dataset_description, language = language)}\n `\n}\n\nfunction LocalCompletions(\n global_explanation: string,\n facts: LocalFactTexts,\n audience_guidance: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n language: string = \"English\",\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, completions_question(num_sentences), facts.completions, audience_guidance, opts.dataset_description, language = language)}\n `\n}\n\n// Entry point called from Python: the three explanations are independent, so they run concurrently.\nfunction explain_local(\n global_explanation: string,\n facts: LocalFactTexts,\n audience_guidance: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n language: string = \"English\",\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> LocalExplanationText {\n let selected = select_client(client, model, short_defaults());\n let counterfactual = spawn {\n run_counted(\n LocalCounterfactual@spec(global_explanation, facts, audience_guidance, num_sentences, opts, language = language),\n client = selected,\n )\n };\n let sufficient = spawn {\n run_counted(\n LocalSufficientReason@spec(global_explanation, facts, audience_guidance, num_sentences, opts, language = language),\n client = selected,\n )\n };\n let completions = spawn {\n run_counted(\n LocalCompletions@spec(global_explanation, facts, audience_guidance, num_sentences, opts, language = language),\n client = selected,\n )\n };\n let done = await baml.future.all([counterfactual, sufficient, completions]);\n LocalExplanationText {\n counterfactual: done[0].text,\n sufficient_reason: done[1].text,\n completions: done[2].text,\n usage: add_usage(add_usage(done[0].usage, done[1].usage), done[2].usage),\n }\n}\n",
26
- "mcp_agent.baml": "// An agent that explains an EBM through an MCP server. `explain_with_mcp` starts the server (the `ebaml-mcp` command),\n// offers its tools to the model, lets the model call them, and returns the final report with the list of calls.\n\n// What the agent returns: the answer, and the tools that it called, in order. A caller checks `tool_calls` to see\n// that the answer rests on tool results.\nclass McpAgentReport {\n report: string,\n tool_calls: string[],\n usage: TokenUsage,\n}\n\nfunction mcp_agent_system_msg() -> string {\n `You are a data scientist who explains a fitted Explainable Boosting Machine (EBM) to a colleague. An EBM is a sum of one graph per feature, so each graph shows exactly how one feature moves the prediction. You reach the model only through tools, and every tool result is exact. Quote only numbers, ranges and effects that a tool returned, because the colleague acts on your report and an invented number would mislead. Say when you did not check something.`\n}\n\nfunction mcp_agent_user_msg(question: string) -> string {\n `\n <instructions>\n 1. Call the tool that summarizes the model, then the tool that lists the features.\n 2. Read the graph of every feature that matters for the question, the most important first.\n 3. If the question concerns one prediction, call the tool that explains a prediction.\n 4. Write the report in plain prose paragraphs. Name the strongest effects with their ranges, and name any surprising or non-monotonic region, because the reader checks it against business knowledge.\n </instructions>\n\n <question>\n ${question}\n </question>\n `\n}\n\n// One message of a chat: who wrote it, and the text.\nclass ChatTurn {\n speaker: \"user\" | \"assistant\",\n text: string,\n}\n\nfunction mcp_chat_system_msg() -> string {\n `${mcp_agent_system_msg()} You chat with the colleague, so answer the latest message and use the earlier messages as context for follow-up questions.`\n}\n\n// The earlier messages as one block of text, so a follow-up such as \"and what if the income were higher?\" has its\n// context. The tool results of earlier turns are not kept, so the model calls a tool again when it needs a number.\nfunction conversation_text(history: ChatTurn[]) -> string {\n history\n .map((turn) -> {\n `<${turn.speaker}>\\n${turn.text}\\n</${turn.speaker}>`\n })\n .join(\"\\n\")\n}\n\nfunction mcp_chat_user_msg(question: string, history: ChatTurn[]) -> string {\n `\n <conversation>\n ${conversation_text(history)}\n </conversation>\n\n <instructions>\n 1. Call only the tools that the latest message needs. Call a tool again if you need a number from an earlier turn.\n 2. To explain one prediction, call the tool that explains a prediction with the number of a row of the data, or with the value of every feature. If the colleague gives neither, ask for one of them.\n 3. Answer in plain prose, in a few sentences, because the colleague reads it in a terminal. Quote only numbers that a tool returned, and round them to three digits, because a longer number hides the point.\n </instructions>\n\n <message>\n ${question}\n </message>\n `\n}\n\n// The agent loop. The model sees the system message and the question, and `ToolRequested` events fill `tool_calls`.\nfunction McpExplainer(question: string) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${mcp_agent_system_msg()}\n ${role(\"user\")}\n ${mcp_agent_user_msg(question)}\n `\n}\n\n// One answer in a chat. `history` holds the messages before `question`.\nfunction McpChat(question: string, history: ChatTurn[]) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${mcp_chat_system_msg()}\n ${role(\"user\")}\n ${mcp_chat_user_msg(question, history)}\n `\n}\n\n// The tool-calling loop for any spec, on tools that the caller supplies. The server gives the tools in\n// `explain_with_mcp` and `chat_with_mcp`, and the offline tests pass tools with fixed results.\nfunction run_with_tools(\n spec: ai.FunctionSpec<string>,\n tools: ai.tools.Tool[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n let tool_calls: string[] = [];\n let samples: TokenSample[] = [];\n let agent = ai.Agent.new(max_steps = max_steps, client = select_client(client, model, short_defaults()), on_event = (\n event,\n ) -> {\n if let call: ai.events.ToolRequested = event {\n tool_calls.push(call.name);\n };\n if let model_call: ai.events.ModelCall = event {\n let sample = sample_of(model_call);\n if (sample != null) {\n samples.push(sample);\n };\n };\n });\n //# The spec carries the tools of the caller. The generated spec has none, because the tools are known at run time.\n let result = agent.run(ai.FunctionSpec { ...spec, toolbox: ai.tools.Toolbox.new(tools) });\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0, samples: samples };\n for (let entry in result.usage) {\n usage\n = add_usage(\n usage,\n TokenUsage {\n input_tokens: entry.usage.input_tokens ?? 0,\n output_tokens: entry.usage.output_tokens ?? 0,\n samples: [],\n },\n );\n }\n McpAgentReport { report: result.value, tool_calls: tool_calls, usage: usage }\n}\n\nfunction explain_with_tools(\n question: string,\n tools: ai.tools.Tool[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n run_with_tools(McpExplainer@spec(question), tools, client = client, model = model, max_steps = max_steps)\n}\n\n// One chat answer on tools that the caller supplies.\nfunction chat_with_tools(\n question: string,\n history: ChatTurn[],\n tools: ai.tools.Tool[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n run_with_tools(McpChat@spec(question, history), tools, client = client, model = model, max_steps = max_steps)\n}\n\n// Entry point called from Python or the command line. `command` and `args` start the MCP server, for example\n// `ebaml-mcp` with `[\"--model\", \"model.skops\", \"--data\", \"train.csv\"]`. The connection closes on every path.\nfunction explain_with_mcp(\n question: string,\n command: string,\n args: string[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n //# Start the server and read its tool catalog.\n let connection = ai.mcp.Connection.connect(\"ebaml\", command, args);\n defer { connection.close() }\n //# Let the model call the tools, then return its report.\n explain_with_tools(question, connection.tools(), client = client, model = model, max_steps = max_steps)\n}\n\n// A chat in the terminal. The server starts once and stays up for the whole chat, so a later question is fast, and a\n// tool that changes the model keeps its change. `baml.io.input` reads end-of-input as an empty line, so an empty line\n// ends the chat. A failed turn (a rate limit, a refused key) prints its error and the chat goes on, because the\n// colleague can ask again.\nfunction chat_with_mcp(\n command: string,\n args: string[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> int {\n //# Start the server once and read its tool catalog.\n let connection = ai.mcp.Connection.connect(\"ebaml\", command, args);\n defer { connection.close() }\n let tools = connection.tools();\n let history: ChatTurn[] = [];\n baml.io.println(\n \"Ask about the model, or about one prediction. Press Enter on an empty line to leave.\",\n );\n //# Read a message, answer it with the tools, and keep both in the history.\n while (true) {\n let message = baml.io.input(\"you> \").trim();\n if (message.length() == 0) {\n break;\n };\n let reply = chat_with_tools(message, history, tools, client = client, model = model, max_steps = max_steps) catch_all (error) {\n _ => {\n McpAgentReport {\n report: `This turn failed: ${error}`,\n tool_calls: [],\n usage: TokenUsage { input_tokens: 0, output_tokens: 0, samples: [] },\n }\n }\n };\n baml.io.println(`\\n${reply.report}\\n`);\n if (reply.tool_calls.length() > 0) {\n baml.io.eprintln(`(tools: ${reply.tool_calls.join(\", \")})`);\n };\n history.push(ChatTurn { speaker: \"user\", text: message });\n history.push(ChatTurn { speaker: \"assistant\", text: reply.report });\n }\n history.length() / 2\n}\n",
26
+ "mcp_agent.baml": "// An agent that explains an EBM through an MCP server. `explain_with_mcp` starts the server (the `ebaml-mcp` command),\n// offers its tools to the model, lets the model call them, and returns the final report with the list of calls.\n\n// What the agent returns: the answer, and the tools that it called, in order. A caller checks `tool_calls` to see\n// that the answer rests on tool results.\nclass McpAgentReport {\n report: string,\n tool_calls: string[],\n usage: TokenUsage,\n}\n\nfunction mcp_agent_system_msg() -> string {\n `You are a data scientist who explains a fitted Explainable Boosting Machine (EBM) to a colleague. An EBM is a sum of one graph per feature, so each graph shows exactly how one feature moves the prediction. You reach the model only through tools, and every tool result is exact. Quote only numbers, ranges and effects that a tool returned, because the colleague acts on your report and an invented number would mislead. Say when you did not check something.`\n}\n\nfunction mcp_agent_user_msg(question: string) -> string {\n `\n <instructions>\n 1. Call the tool that summarizes the model, then the tool that lists the features.\n 2. Read the graph of every feature that matters for the question, the most important first.\n 3. If the question concerns one prediction, call the tool that explains a prediction.\n 4. Write the report in plain prose paragraphs. Name the strongest effects with their ranges, and name any surprising or non-monotonic region, because the reader checks it against business knowledge.\n </instructions>\n\n <question>\n ${question}\n </question>\n `\n}\n\n// One message of a chat: who wrote it, and the text.\nclass ChatTurn {\n speaker: \"user\" | \"assistant\",\n text: string,\n}\n\nfunction mcp_chat_system_msg() -> string {\n `${mcp_agent_system_msg()} You chat with the colleague, so answer the latest message and use the earlier messages as context for follow-up questions.`\n}\n\n// The earlier messages as one block of text, so a follow-up such as \"and what if the income were higher?\" has its\n// context. The tool results of earlier turns are not kept, so the model calls a tool again when it needs a number.\nfunction conversation_text(history: ChatTurn[]) -> string {\n history\n .map((turn) -> {\n `<${turn.speaker}>\\n${turn.text}\\n</${turn.speaker}>`\n })\n .join(\"\\n\")\n}\n\nfunction mcp_chat_user_msg(question: string, history: ChatTurn[]) -> string {\n `\n <conversation>\n ${conversation_text(history)}\n </conversation>\n\n <instructions>\n 1. Call only the tools that the latest message needs. Call a tool again if you need a number from an earlier turn.\n 2. To explain one prediction, call the tool that explains a prediction with the number of a row of the data, or with the value of every feature. If the colleague gives neither, ask for one of them.\n 3. Answer in plain prose, in a few sentences, because the colleague reads it in a terminal. Quote only numbers that a tool returned, and round them to three digits, because a longer number hides the point.\n </instructions>\n\n <message>\n ${question}\n </message>\n `\n}\n\n// The agent loop. The model sees the system message and the question, and `ToolRequested` events fill `tool_calls`.\nfunction McpExplainer(question: string) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${mcp_agent_system_msg()}\n ${role(\"user\")}\n ${mcp_agent_user_msg(question)}\n `\n}\n\n// One answer in a chat. `history` holds the messages before `question`.\nfunction McpChat(question: string, history: ChatTurn[]) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${mcp_chat_system_msg()}\n ${role(\"user\")}\n ${mcp_chat_user_msg(question, history)}\n `\n}\n\n// The tool-calling loop for any spec, on tools that the caller supplies. The server gives the tools in\n// `explain_with_mcp` and `chat_with_mcp`, and the offline tests pass tools with fixed results.\nfunction run_with_tools(\n spec: ai.FunctionSpec<string>,\n tools: ai.tools.Tool[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n let tool_calls: string[] = [];\n let samples: TokenSample[] = [];\n let agent = ai.Agent.new(max_steps = max_steps, client = select_client(client, model, short_defaults()), on_event = (\n event,\n ) -> {\n if let call: ai.events.ToolRequested = event {\n tool_calls.push(call.name);\n };\n if let model_call: ai.events.ModelCall = event {\n let sample = sample_of(model_call);\n if (sample != null) {\n samples.push(sample);\n };\n };\n });\n //# The spec carries the tools of the caller. The generated spec has none, because the tools are known at run time.\n let result = agent.run(ai.FunctionSpec { ...spec, toolbox: ai.tools.Toolbox.new(tools) });\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0, samples: samples };\n for (let entry in result.usage) {\n usage\n = add_usage(\n usage,\n TokenUsage {\n input_tokens: entry.usage.input_tokens ?? 0,\n output_tokens: entry.usage.output_tokens ?? 0,\n samples: [],\n },\n );\n }\n McpAgentReport { report: result.value, tool_calls: tool_calls, usage: usage }\n}\n\nfunction explain_with_tools(\n question: string,\n tools: ai.tools.Tool[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n run_with_tools(McpExplainer@spec(question), tools, client = client, model = model, max_steps = max_steps)\n}\n\n// One chat answer on tools that the caller supplies.\nfunction chat_with_tools(\n question: string,\n history: ChatTurn[],\n tools: ai.tools.Tool[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n run_with_tools(McpChat@spec(question, history), tools, client = client, model = model, max_steps = max_steps)\n}\n\n// Entry point called from Python or the command line. `command` and `args` start the MCP server, for example\n// `ebaml-mcp` with `[\"--model\", \"model.skops\", \"--data\", \"train.parquet\"]`. The connection closes on every path.\nfunction explain_with_mcp(\n question: string,\n command: string,\n args: string[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> McpAgentReport {\n //# Start the server and read its tool catalog.\n let connection = ai.mcp.Connection.connect(\"ebaml\", command, args);\n defer { connection.close() }\n //# Let the model call the tools, then return its report.\n explain_with_tools(question, connection.tools(), client = client, model = model, max_steps = max_steps)\n}\n\n// A chat in the terminal. The server starts once and stays up for the whole chat, so a later question is fast, and a\n// tool that changes the model keeps its change. `baml.io.input` reads end-of-input as an empty line, so an empty line\n// ends the chat. A failed turn (a rate limit, a refused key) prints its error and the chat goes on, because the\n// colleague can ask again.\nfunction chat_with_mcp(\n command: string,\n args: string[],\n client: ai.Client? = null,\n model: ModelSpec? = null,\n max_steps: int = 12,\n) -> int {\n //# Start the server once and read its tool catalog.\n let connection = ai.mcp.Connection.connect(\"ebaml\", command, args);\n defer { connection.close() }\n let tools = connection.tools();\n let history: ChatTurn[] = [];\n baml.io.println(\n \"Ask about the model, or about one prediction. Press Enter on an empty line to leave.\",\n );\n //# Read a message, answer it with the tools, and keep both in the history.\n while (true) {\n let message = baml.io.input(\"you> \").trim();\n if (message.length() == 0) {\n break;\n };\n let reply = chat_with_tools(message, history, tools, client = client, model = model, max_steps = max_steps) catch_all (error) {\n _ => {\n McpAgentReport {\n report: `This turn failed: ${error}`,\n tool_calls: [],\n usage: TokenUsage { input_tokens: 0, output_tokens: 0, samples: [] },\n }\n }\n };\n baml.io.println(`\\n${reply.report}\\n`);\n if (reply.tool_calls.length() > 0) {\n baml.io.eprintln(`(tools: ${reply.tool_calls.join(\", \")})`);\n };\n history.push(ChatTurn { speaker: \"user\", text: message });\n history.push(ChatTurn { speaker: \"assistant\", text: reply.report });\n }\n history.length() / 2\n}\n",
27
27
  "model_spec.baml": "// A model chosen at run time. Python sends a `ModelSpec` (plain data) and BAML builds the client, because a client\n// object cannot cross the Python/BAML boundary.\n\nclass ModelSpec {\n provider: string,\n model: string,\n api_key_env: string?,\n base_url: string?,\n temperature: float?,\n max_output_tokens: int?,\n headers: map<string, string>?,\n}\n\n// The settings of one call: used for each field that the `ModelSpec` leaves null.\nclass CallDefaults {\n temperature: float,\n max_output_tokens: int,\n}\n\n// These three must equal the `temperature` and `max_tokens` of `RouterBase`, `RouterLongBase` and\n// `RouterBenchmarkBase` in talk_to_ebm.baml.\nfunction short_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens: 12000 }\n}\n\n// The first turn of each conversation allows a longer answer.\nfunction long_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens: 16000 }\n}\n\nfunction benchmark_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.0, max_output_tokens: 16000 }\n}\n\n// A field that the user set wins. Otherwise the default of the call applies.\nfunction resolve_settings(spec: ModelSpec, defaults: CallDefaults) -> CallDefaults {\n CallDefaults {\n temperature: spec.temperature ?? defaults.temperature,\n max_output_tokens: spec.max_output_tokens ?? defaults.max_output_tokens,\n }\n}\n\n// The key is read from the environment when the request is sent. Null selects the default variable of the provider.\nfunction credential(name: string?) -> ai.Credential? {\n if (name == null) {\n return null;\n };\n baml.env.ref(name)\n}\n\nfunction build_openai(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.ChatClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_openai_generic(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.GenericClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// Azure serves deployments under names that the owner chose, so `model` names the deployment. `base_url` names it too\n// and wins. Without it, the deployment is built on the endpoint in `AZURE_OPENAI_ENDPOINT`.\nfunction build_azure(spec: ModelSpec, settings: CallDefaults) -> openai.AzureClient {\n openai.AzureClient.new(\n model = spec.model,\n deployment_id = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_openrouter(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.OpenRouterClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_ollama(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.OllamaClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_anthropic(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n anthropic.Client.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// Gemini names its output limit `max_output_tokens`.\nfunction build_google(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n google.GeminiClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_output_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_vertex(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n google.VertexClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_output_tokens = settings.max_output_tokens,\n )\n}\n\n// Bedrock signs requests with AWS credentials, so `api_key_env` has no effect.\nfunction build_bedrock(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n aws.BedrockClient.new(\n model = spec.model,\n endpoint_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// One row for each provider. A new provider is a new row. The keys are the BAML provider prefixes.\nfunction client_builders() -> map<string, (ModelSpec, CallDefaults) -> ai.Client throws never> {\n {\n \"openai\": build_openai,\n \"openai-generic\": build_openai_generic,\n \"azure\": build_azure,\n \"openrouter\": build_openrouter,\n \"ollama\": build_ollama,\n \"anthropic\": build_anthropic,\n \"google\": build_google,\n \"vertex\": build_vertex,\n \"bedrock\": build_bedrock,\n }\n}\n\n// The provider prefixes that `ModelSpec.provider` accepts, in table order.\nfunction supported_providers() -> string[] {\n client_builders().keys()\n}\n\n// Builds the client of `spec` with the retry policy of the default clients.\nfunction build_client(spec: ModelSpec, defaults: CallDefaults) -> ai.Client {\n let builders = client_builders();\n let build = builders.get(spec.provider);\n if (build == null) {\n throw baml.errors.InvalidArgument {\n message: `unknown provider \"${spec.provider}\"; supported providers: ${supported_providers().join(\", \")}`,\n };\n };\n ai.clients.Retry.new(inner = build(spec, resolve_settings(spec, defaults)), max_attempts = 3)\n}\n\n// Builds and discards the client, so that Python can reject a bad provider before the first paid call.\nfunction check_model(spec: ModelSpec) -> bool {\n build_client(spec, short_defaults());\n true\n}\n\n// Precedence: `client` (the test hook) > `model` > null, which selects the client declared by the function.\nfunction select_client(\n client: ai.Client?,\n model: ModelSpec?,\n defaults: CallDefaults,\n) -> ai.Client? {\n if (client != null) {\n return client;\n };\n if (model == null) {\n return null;\n };\n build_client(model, defaults)\n}\n",
28
28
  "talk_to_ebm.baml": "// Port of TalkToEBM's t2ebm/prompts.py and t2ebm/llm.py: prompts, chat clients, CoT pipelines, token accounting.\n\n// Some models behind `openrouter/free` reason mandatorily (reasoning cannot be disabled), and the hidden reasoning tokens\n// count against `max_tokens`, so the caps are generous. 16000 stays below the output cap of 16384 tokens of common OpenAI\n// models, because these limits are also the defaults of a ModelSpec for any provider.\n// Transient failures (rate limits, network, 5xx) are retried; rejected requests are not.\nclient RouterBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 12000);\n\n// First assistant turn of each conversation allows a longer answer.\nclient RouterLongBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 16000);\n\nclient Router = ai.clients.Retry.new(inner = RouterBase, max_attempts = 3);\n\n// The benchmark is scored on one answer per question, so it samples greedily, as TalkToEBM's benchmark does.\nclient RouterBenchmarkBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.0, max_tokens = 16000);\n\nclient RouterBenchmark = ai.clients.Retry.new(inner = RouterBenchmarkBase, max_attempts = 3);\n\nclient RouterLong = ai.clients.Retry.new(inner = RouterLongBase, max_attempts = 3);\n\nclass GraphPromptOptions {\n expert_description: string,\n graph_description: string,\n dataset_description: string,\n task_description: string,\n}\n\nfunction default_graph_options() -> GraphPromptOptions {\n GraphPromptOptions {\n expert_description: \"an expert statistician and data scientist\",\n graph_description: \"\",\n dataset_description: \"\",\n task_description: \"Describe the general pattern of the graph.\",\n }\n}\n\n// Prompt text helpers. Pure functions, so the Jinja `{% if %}` blocks become plain BAML.\n\nfunction graph_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You interpret global explanations produced by a Generalized Additive Model (GAM). Base every answer on the graph from the user, on any other information you are given, and on your knowledge of the real world. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// How to read the series of the classes. Only the graph of a multiclass classifier has \"Class: \" lines, so the prompt\n// of a binary classifier or a regressor gets no text about classes.\nfunction multiclass_format(graph: string) -> string {\n if (graph.includes(\"Class: \")) {\n return \"\\nA graph of a multiclass classifier repeats items 3 to 5 once for each class, after a line \\\"Class: <name>\\\". In every bin, the scores have the same mean over all the classes of the model, including classes that the graph leaves out. Only the differences between the classes change the predicted probabilities, so compare the classes with each other within each bin.\";\n };\n \"\"\n}\n\nfunction describe_graph_msg(graph: string, opts: GraphPromptOptions) -> string {\n let msg = `\n <graph_format>\n The graph is a JSON object. Its keys are the x-axis and its values are the y-axis. For a continuous feature, each key is an interval where the function predicts the same value. For a categorical feature, each key is a possible value of the feature.\n The graph contains these items:\n 1. The name of the feature.\n 2. The type of the feature (continuous, categorical, or boolean).\n 3. The mean values.\n 4. The lower bounds of the confidence interval (optional).\n 5. The upper bounds of the confidence interval (optional).${multiclass_format(graph)}\n </graph_format>\n\n <graph>\n ${graph}\n </graph>\n `\n + \"\\n\\n\";\n if (opts.graph_description.length() > 0) {\n msg = msg + `<graph_description>\\n${opts.graph_description}\\n</graph_description>\\n\\n`;\n };\n if (opts.dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${opts.dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg + `<task>\\n${opts.task_description}\\n</task>`\n}\n\n// The parts of the model that a description leaves out. The reader of a description cannot see a log warning, so the\n// text itself must name them. An empty list gives no block, so a model without such parts gets no text about them.\nfunction omitted_parts_msg(omitted_terms: string[], omitted_features: string[]) -> string {\n if (omitted_terms.length() + omitted_features.length() == 0) {\n return \"\";\n };\n let msg = \"<omitted_parts>\\nThe graph descriptions cover the main effect of one feature at a time.\\n\";\n if (omitted_terms.length() > 0) {\n msg\n = msg\n + `The model also has these interaction terms, and no graph description covers them: ${omitted_terms.join(\", \")}.\\n`;\n };\n if (omitted_features.length() > 0) {\n msg\n = msg\n + `These features have no main effect in the model, and no graph description covers them: ${omitted_features.join(\", \")}.\\n`;\n };\n msg\n + \"Say in one sentence that your answer leaves these parts out, and name each of them, because the reader must know that the answer does not cover the whole model.\\n</omitted_parts>\\n\\n\"\n}\n\nfunction summarize_ebm_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You provide an overall summary of a Generalized Additive Model (GAM). The model consists of different graphs, and each graph shows the effect of one input feature. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\nfunction summarize_ebm_msg(\n feature_importances: string,\n graph_descriptions: string,\n dataset_description: string,\n omitted_terms: string[] = [],\n omitted_features: string[] = [],\n) -> string {\n let msg = `\n <feature_importances>\n ${feature_importances}\n </feature_importances>\n\n <graph_descriptions>\n ${graph_descriptions}\n </graph_descriptions>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + omitted_parts_msg(omitted_terms, omitted_features)\n + `\n <instructions>\n Summarize the model. The feature importances are global, and there is exactly one graph description for each feature.\n Cover the most important features and their effect on the outcome. Leave out unimportant features and effects, so the summary stays short.\n Include any surprising pattern, because the reader wants to know where the model differs from expectation.\n </instructions>\n `\n}\n\n// Request text shared by the second turn and the replay of that turn in the third.\nfunction surprises_request() -> string {\n `\n <instructions>\n Study the graph carefully. Highlight any region that you find surprising or counterintuitive, and name its range or category so the reader can find it. Suggest why the behavior is surprising and what may have caused it.\n </instructions>\n `\n}\n\nfunction final_description_request(num_sentences: int, omitted_terms: string[] = []) -> string {\n omitted_parts_msg(omitted_terms, [])\n + `\n <instructions>\n Describe the graph in at most ${num_sentences} sentences.\n Include the important surprising patterns, because the reader uses them to decide where to look closer.\n The reader knows that the graph is from a Generalized Additive Model (GAM), so leave that background out.\n </instructions>\n `\n}\n\n// describe_graph_cot, turn 1: system + describe_graph prompt.\nfunction GraphFirstPass(graph: string, opts: GraphPromptOptions) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n `\n}\n\n// Turn 2: surprising regions. `first_pass` is the assistant reply from turn 1.\nfunction GraphSurprises(graph: string, first_pass: string, opts: GraphPromptOptions) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n `\n}\n\n// Turn 3: brief final description.\nfunction GraphFinalDescription(\n graph: string,\n first_pass: string,\n surprises: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n omitted_terms: string[] = [],\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n ${role(\"assistant\")}\n ${surprises}\n ${role(\"user\")}\n ${final_description_request(num_sentences, omitted_terms = omitted_terms)}\n `\n}\n\n// summarize_ebm, turn 1.\nfunction EbmSummaryDraft(\n feature_importances: string,\n graph_descriptions: string,\n opts: GraphPromptOptions,\n omitted_terms: string[] = [],\n omitted_features: string[] = [],\n) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description, omitted_terms = omitted_terms, omitted_features = omitted_features)}\n `\n}\n\n// summarize_ebm, turn 2: shorten the draft.\nfunction EbmSummaryShorten(\n feature_importances: string,\n graph_descriptions: string,\n draft: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n omitted_terms: string[] = [],\n omitted_features: string[] = [],\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description, omitted_terms = omitted_terms, omitted_features = omitted_features)}\n ${role(\"assistant\")}\n ${draft}\n ${role(\"user\")}\n <instructions>\n Shorten your summary to at most ${num_sentences} sentences. Keep the most important information, because the reader wants the main effects and surprises. If the input has an <omitted_parts> block, keep the sentence that names the parts that the summary leaves out, because the reader must know that the summary does not cover the whole model.\n </instructions>\n `\n}\n\n// Token accounting.\n\n// One request that the provider counted. Python reads the text of the messages in `body`, takes the prose away and\n// fits the weights of its token estimate to the tokens of the graph that is left. The sample holds the request body\n// only: the headers carry the API key, so they never leave the runtime.\nclass TokenSample {\n provider: string,\n model: string,\n body: string,\n input_tokens: int,\n}\n\nclass TokenUsage {\n input_tokens: int,\n output_tokens: int,\n samples: TokenSample[],\n}\n\n// An LLM result plus the tokens the provider reported for producing it.\nclass Counted {\n text: string,\n usage: TokenUsage,\n}\n\nfunction add_usage(a: TokenUsage, b: TokenUsage) -> TokenUsage {\n TokenUsage {\n input_tokens: a.input_tokens + b.input_tokens,\n output_tokens: a.output_tokens + b.output_tokens,\n samples: a.samples.concat(b.samples),\n }\n}\n\n// The input tokens of one call, with the cached and the cache-write tokens included, or null when the provider\n// reported no count. A count that is missing is not zero: a zero would pull the weights of the estimate toward zero.\nfunction input_total(usage: ai.events.Usage) -> int? {\n let uncached = usage.uncached_input_tokens;\n let cached = usage.cached_input_tokens;\n let written = usage.cache_write_input_tokens;\n if (uncached != null || cached != null || written != null) {\n return (uncached ?? 0) + (cached ?? 0) + (written ?? 0);\n };\n usage.input_tokens\n}\n\n// The sample of one model call. A call with no request, no usage, no model name or no input count gives no sample.\nfunction sample_of(call: ai.events.ModelCall) -> TokenSample? {\n let usage = call.usage;\n let request = call.http_request;\n let model = call.resolved_model.model;\n if (usage == null || request == null || model == null) {\n return null;\n };\n let total = input_total(usage);\n if (total == null || total < 1) {\n return null;\n };\n TokenSample {\n provider: call.resolved_model.provider,\n model: model,\n body: request.body,\n input_tokens: total,\n }\n}\n\n// The samples of every model call in the events of a run, retries included.\nfunction samples_of(events: ai.events.Event[]) -> TokenSample[] {\n let samples: TokenSample[] = [];\n for (let event in events) {\n if let call: ai.events.ModelCall = event {\n let sample = sample_of(call);\n if (sample != null) {\n samples.push(sample);\n };\n };\n }\n samples\n}\n\n// Runs one text-producing LLM function and keeps the provider's token usage. A count that the provider did not report\n// is zero.\nfunction run_counted(spec: ai.FunctionSpec<string>, client: ai.Client? = null) -> Counted {\n let result = ai.Agent.new(client = client).run(spec);\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0, samples: samples_of(result.journal.entries()) };\n for (let entry in result.usage) {\n usage\n = add_usage(\n usage,\n TokenUsage {\n input_tokens: entry.usage.input_tokens ?? 0,\n output_tokens: entry.usage.output_tokens ?? 0,\n samples: [],\n },\n );\n }\n Counted { text: result.value, usage: usage }\n}\n\n// Entry points called from Python.\n\n// Sends the first request of the graph description chain and returns the token samples of the requests. The sample\n// holds the tokens of the input, so a short reply limit in `model` (`max_output_tokens`) keeps the cost low. A reply\n// that the limit cuts off is not an error here, and the samples of the requests that were sent still come back. Any\n// other error propagates. The scripts use the function to fit the weights of the token estimate of a model.\nfunction sample_graph_tokens(\n graph: string,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> TokenSample[] {\n let samples: TokenSample[] = [];\n let agent = ai.Agent.new(client = select_client(client, model, long_defaults()), on_event = (event) -> {\n if let call: ai.events.ModelCall = event {\n let sample = sample_of(call);\n if (sample != null) {\n samples.push(sample);\n };\n };\n });\n //# A reply that hits the limit raises. The input counts of the request are what this function returns.\n //# Any other error (a rate limit, a refused key) is not a cut-off reply, so it propagates.\n let finished = {\n agent.run(GraphFirstPass@spec(graph, opts));\n true\n } catch (e) {\n ai.errors.FinishReasonError => false\n };\n samples\n}\n\nfunction describe_graph_cot(\n graph: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n omitted_terms: string[] = [],\n) -> Counted {\n //# Three sequential assistant turns, each seeing the previous replies\n let long = select_client(client, model, long_defaults());\n let short = select_client(client, model, short_defaults());\n let first_pass = run_counted(GraphFirstPass@spec(graph, opts), client = long);\n let surprises = run_counted(GraphSurprises@spec(graph, first_pass.text, opts), client = short);\n let final = run_counted(\n GraphFinalDescription@spec(graph, first_pass.text, surprises.text, num_sentences, opts, omitted_terms = omitted_terms),\n client = short,\n );\n Counted {\n text: final.text,\n usage: add_usage(add_usage(first_pass.usage, surprises.usage), final.usage),\n }\n}\n\n// `num_sentences` null keeps the draft summary as is.\nfunction summarize_ebm(\n feature_importances: string,\n graph_descriptions: string,\n num_sentences: int?,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n omitted_terms: string[] = [],\n omitted_features: string[] = [],\n) -> Counted {\n let draft = run_counted(\n EbmSummaryDraft@spec(\n feature_importances,\n graph_descriptions,\n opts,\n omitted_terms = omitted_terms,\n omitted_features = omitted_features,\n ),\n client = select_client(client, model, long_defaults()),\n );\n if (num_sentences == null) {\n return draft;\n };\n let shortened = run_counted(\n EbmSummaryShorten@spec(\n feature_importances,\n graph_descriptions,\n draft.text,\n num_sentences,\n opts,\n omitted_terms = omitted_terms,\n omitted_features = omitted_features,\n ),\n client = select_client(client, model, short_defaults()),\n );\n Counted { text: shortened.text, usage: add_usage(draft.usage, shortened.usage) }\n}\n\n// Full EBM description: one chain per feature graph (concurrently), then the summary.\n// `graphs[i]` is the text of the graph of `feature_names[i]`. The summary names `omitted_terms` and `omitted_features`.\nfunction describe_ebm(\n feature_names: string[],\n graphs: string[],\n feature_importances: string,\n num_sentences: int,\n graph_sentences: int,\n max_concurrency: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n omitted_terms: string[] = [],\n omitted_features: string[] = [],\n) -> Counted {\n if (feature_names.length() != graphs.length()) {\n throw baml.errors.InvalidArgument { message: \"feature_names and graphs must have the same length\" };\n };\n //# One description chain per graph, at most `max_concurrency` at a time\n let limit = baml.spawn.Limit.new(max_concurrency);\n let futures = graphs.map((graph) -> {\n spawn with limit {\n describe_graph_cot(graph, graph_sentences, opts, client = client, model = model)\n }\n });\n let described: Counted[] = await baml.future.all(futures);\n\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0, samples: [] };\n let lines: string[] = [];\n let i = 0;\n while (i < described.length()) {\n let item = described[i];\n usage = add_usage(usage, item.usage);\n lines.push(`${feature_names[i]}: ${item.text}`);\n i = i + 1;\n }\n //# Summarize all the per-graph descriptions\n let summary = summarize_ebm(\n feature_importances,\n lines.join(\"\\n\\n\"),\n num_sentences,\n opts,\n client = client,\n model = model,\n omitted_terms = omitted_terms,\n omitted_features = omitted_features,\n );\n Counted { text: summary.text, usage: add_usage(usage, summary.usage) }\n}\n\n// Benchmark (benchmark/): the questions end with \"Think step by step.\", so turn 1 reasons freely and turn 2\n// extracts the final answer in the format the scorer parses.\n\nfunction BenchmarkReasoning(question: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n `\n}\n\nfunction BenchmarkAnswer(question: string, reasoning: string, answer_format: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n ${role(\"assistant\")}\n ${reasoning}\n ${role(\"user\")}\n <instructions>\n Summarize your response by answering with ${answer_format}. Reply with the answer only, because a program parses your reply.\n </instructions>\n `\n}\n\nfunction answer_benchmark_question(\n question: string,\n answer_format: string,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let selected = select_client(client, model, benchmark_defaults());\n let reasoning = run_counted(BenchmarkReasoning@spec(question), client = selected);\n let answer = run_counted(BenchmarkAnswer@spec(question, reasoning.text, answer_format), client = selected);\n Counted { text: answer.text, usage: add_usage(reasoning.usage, answer.usage) }\n}\n",
29
29
  "tests/graph_description_test.baml": "// Offline tests of the graph description chain, the summary, describe_ebm and the benchmark answer in talk_to_ebm.baml, on scripted clients.\n\ntest \"describe_graph_msg includes optional sections only when set\" {\n let bare = describe_graph_msg(\"GRAPH\", default_graph_options());\n assert.is_true(bare.includes(\"GRAPH\"));\n assert.is_true(bare.includes(\"Describe the general pattern of the graph.\"));\n assert.is_true(!bare.includes(\"<dataset_description>\"));\n\n let opts = GraphPromptOptions { ...default_graph_options(), dataset_description: \"A pneumonia dataset\" };\n assert.is_true(describe_graph_msg(\"GRAPH\", opts).includes(\"A pneumonia dataset\"))\n}\n\ntest \"describe_graph_msg says how to read the classes of a multiclass graph\" {\n let msg = describe_graph_msg(\"Feature Name: a\\nClass: high\\nMeans: {}\", default_graph_options());\n assert.is_true(\n msg.includes(\"(optional).\\nA graph of a multiclass classifier repeats items 3 to 5\"),\n );\n assert.is_true(msg.includes(\"including classes that the graph leaves out\"));\n assert.is_true(msg.includes(\"compare the classes with each other within each bin.\"));\n assert.is_true(position_of(msg, \"Class: <name>\") < position_of(msg, \"</graph_format>\"))\n}\n\ntest \"describe_graph_msg of a graph with one output says nothing about classes\" {\n let msg = describe_graph_msg(\"Feature Name: a\\nMeans: {}\", default_graph_options());\n assert.is_true(!msg.includes(\"multiclass\"));\n assert.is_true(\n msg.includes(\"5. The upper bounds of the confidence interval (optional).\\n</graph_format>\"),\n )\n}\n\ntest \"describe_graph_cot makes three calls and sums usage\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = scripted);\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 3);\n assert.equal(result.usage.input_tokens, 30);\n assert.equal(result.usage.output_tokens, 15)\n}\n\ntest \"summarize_ebm shortens only when num_sentences is set\" {\n let draft_only = ScriptedClient { calls: 0 };\n summarize_ebm(\"IMP\", \"DESC\", null, default_graph_options(), client = draft_only);\n assert.equal(draft_only.calls, 1);\n\n let shortened = ScriptedClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = shortened);\n assert.equal(shortened.calls, 2);\n assert.equal(result.usage.input_tokens, 20)\n}\n\ntest \"describe_ebm: one chain per graph plus the summary\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_ebm([\"age\", \"bmi\"], [\"G1\", \"G2\"], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted);\n // 2 graphs x 3 turns + draft + shorten. The two chains run at the same time, so `self.calls += 1` of the\n // scripted client can lose an increment; each call reports 10 input and 5 output tokens, so the usage counts the\n // calls without a shared counter.\n assert.equal(result.usage.input_tokens, 8 * 10);\n assert.equal(result.usage.output_tokens, 8 * 5)\n}\n\ntest \"describe_ebm rejects mismatched names and graphs\" {\n let scripted = ScriptedClient { calls: 0 };\n let outcome = describe_ebm([\"age\"], [], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted) catch (e) {\n baml.errors.InvalidArgument => null,\n };\n assert.is_true(outcome == null);\n assert.equal(scripted.calls, 0)\n}\n\ntest \"answer_benchmark_question reasons, then extracts the answer\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = scripted);\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 2);\n assert.equal(result.usage.input_tokens, 20);\n assert.equal(result.usage.output_tokens, 10)\n}\n\n// The roles of a rendered prompt, joined, e.g. \"system,user,assistant,user\".\nfunction roles_of(prompt: ai.Prompt) -> string {\n prompt.messages()\n .map((m) -> {\n m.role\n })\n .join(\",\")\n}\n\n// The content of message `index` of a rendered prompt.\nfunction message_at(prompt: ai.Prompt, index: int) -> string {\n prompt.messages()[index].content\n}\n\ntest \"default_graph_options is a plain statistician with no extra context\" {\n let opts = default_graph_options();\n assert.equal(opts.expert_description, \"an expert statistician and data scientist\");\n assert.equal(opts.graph_description, \"\");\n assert.equal(opts.dataset_description, \"\");\n assert.equal(opts.task_description, \"Describe the general pattern of the graph.\")\n}\n\ntest \"graph_system_msg names the expert and the GAM\" {\n let msg = graph_system_msg(\"a medical statistician\");\n assert.is_true(msg.includes(\"You are a medical statistician.\"));\n assert.is_true(msg.includes(\"Generalized Additive Model\"))\n}\n\ntest \"describe_graph_msg orders graph, graph description, dataset and task\" {\n let opts = GraphPromptOptions {\n expert_description: \"E\",\n graph_description: \"GRAPHDESC\",\n dataset_description: \"DATASET\",\n task_description: \"TASK\",\n };\n let msg = describe_graph_msg(\"GRAPH\", opts);\n assert.is_true(position_of(msg, \"GRAPH\") >= 0);\n assert.is_true(position_of(msg, \"GRAPH\") < position_of(msg, \"GRAPHDESC\"));\n assert.is_true(position_of(msg, \"GRAPHDESC\") < position_of(msg, \"DATASET\"));\n assert.is_true(position_of(msg, \"DATASET\") < position_of(msg, \"TASK\"));\n assert.is_true(msg.includes(\"<dataset_description>\"))\n}\n\ntest \"describe_graph_msg leaves out each optional section on its own\" {\n let only_graph = GraphPromptOptions { ...default_graph_options(), graph_description: \"GRAPHDESC\" };\n let msg = describe_graph_msg(\"GRAPH\", only_graph);\n assert.is_true(msg.includes(\"GRAPHDESC\"));\n assert.is_true(!msg.includes(\"<dataset_description>\"));\n\n let only_dataset = GraphPromptOptions { ...default_graph_options(), dataset_description: \"DATASET\" };\n let other = describe_graph_msg(\"GRAPH\", only_dataset);\n assert.is_true(other.includes(\"DATASET\"));\n assert.is_true(!other.includes(\"GRAPHDESC\"));\n\n let custom = GraphPromptOptions { ...default_graph_options(), task_description: \"MY TASK\" };\n let task = describe_graph_msg(\"GRAPH\", custom);\n assert.is_true(task.includes(\"MY TASK\"));\n assert.is_true(!task.includes(\"Describe the general pattern\"))\n}\n\ntest \"summarize_ebm_system_msg names the expert\" {\n let msg = summarize_ebm_system_msg(\"a risk analyst\");\n assert.is_true(msg.includes(\"You are a risk analyst.\"));\n assert.is_true(msg.includes(\"overall summary\"))\n}\n\ntest \"summarize_ebm_msg orders importances, descriptions and dataset, and the dataset is optional\" {\n let msg = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"DATASET\");\n assert.is_true(position_of(msg, \"IMPORTANCES\") >= 0);\n assert.is_true(position_of(msg, \"IMPORTANCES\") < position_of(msg, \"DESCRIPTIONS\"));\n assert.is_true(position_of(msg, \"DESCRIPTIONS\") < position_of(msg, \"DATASET\"));\n assert.is_true(position_of(msg, \"DATASET\") < position_of(msg, \"<instructions>\"));\n\n let bare = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"\");\n assert.is_true(!bare.includes(\"<dataset_description>\"));\n assert.is_true(bare.includes(\"<instructions>\"))\n}\n\ntest \"describe_graph_cot prompts: roles, content and sentence limit per turn\" {\n let opts = GraphPromptOptions { ...default_graph_options(), expert_description: \"an EXPERT\" };\n\n let first = GraphFirstPass@spec(\"GRAPH\", opts).prompt();\n assert.equal(roles_of(first), \"system,user\");\n assert.is_true(message_at(first, 0).includes(\"You are an EXPERT.\"));\n assert.is_true(message_at(first, 1).includes(\"GRAPH\"));\n\n let second = GraphSurprises@spec(\"GRAPH\", \"FIRST\", opts).prompt();\n assert.equal(roles_of(second), \"system,user,assistant,user\");\n assert.is_true(message_at(second, 1).includes(\"GRAPH\"));\n assert.equal(message_at(second, 2).trim(), \"FIRST\");\n assert.is_true(message_at(second, 3).includes(\"surprising or counterintuitive\"));\n\n let third = GraphFinalDescription@spec(\"GRAPH\", \"FIRST\", \"SURPRISES\", 7, opts).prompt();\n assert.equal(roles_of(third), \"system,user,assistant,user,assistant,user\");\n assert.equal(message_at(third, 2).trim(), \"FIRST\");\n assert.equal(message_at(third, 4).trim(), \"SURPRISES\");\n assert.is_true(message_at(third, 5).includes(\"at most 7 sentence\"))\n}\n\ntest \"describe_graph_cot feeds each reply to the next turn in order\" {\n let sequence = SequenceClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = sequence);\n assert.equal(sequence.calls, 3);\n assert.equal(result.text, \"REPLY-3\");\n assert.equal(result.usage.input_tokens, 3);\n assert.equal(result.usage.output_tokens, 3);\n\n // The echo client returns the last prompt, which must hold both earlier replies in order.\n let echo = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = EchoClient {});\n assert.is_true(echo.text.includes(\"GRAPH\"));\n assert.is_true(echo.text.includes(\"at most 7 sentence\"));\n assert.is_true(position_of(echo.text, \"Highlight any region\") >= 0);\n assert.is_true(\n position_of(echo.text, \"Describe the graph in at most\")\n > position_of(echo.text, \"Highlight any region\"),\n )\n}\n\ntest \"describe_graph_cot passes the second and third turns the replies of the earlier turns\" {\n let sequence = SequenceClient { calls: 0 };\n let first = GraphFirstPass@spec(\"GRAPH\", default_graph_options());\n let reply_one = run_counted(first, client = sequence);\n assert.equal(reply_one.text, \"REPLY-1\");\n let reply_two = run_counted(GraphSurprises@spec(\"GRAPH\", reply_one.text, default_graph_options()), client = sequence);\n assert.equal(reply_two.text, \"REPLY-2\");\n let third = GraphFinalDescription@spec(\"GRAPH\", reply_one.text, reply_two.text, 5, default_graph_options())\n .prompt();\n assert.equal(message_at(third, 2).trim(), \"REPLY-1\");\n assert.equal(message_at(third, 4).trim(), \"REPLY-2\")\n}\n\ntest \"summarize_ebm prompts: roles, content and sentence limit\" {\n let opts = GraphPromptOptions {\n ...default_graph_options(),\n expert_description: \"an EXPERT\",\n dataset_description: \"DATASET\",\n };\n let draft = EbmSummaryDraft@spec(\"IMPORTANCES\", \"DESCRIPTIONS\", opts).prompt();\n assert.equal(roles_of(draft), \"system,user\");\n assert.is_true(message_at(draft, 0).includes(\"You are an EXPERT.\"));\n assert.is_true(message_at(draft, 1).includes(\"IMPORTANCES\"));\n assert.is_true(message_at(draft, 1).includes(\"DESCRIPTIONS\"));\n assert.is_true(message_at(draft, 1).includes(\"DATASET\"));\n\n let shorten = EbmSummaryShorten@spec(\"IMPORTANCES\", \"DESCRIPTIONS\", \"DRAFT\", 12, opts).prompt();\n assert.equal(roles_of(shorten), \"system,user,assistant,user\");\n assert.is_true(message_at(shorten, 1).includes(\"IMPORTANCES\"));\n assert.equal(message_at(shorten, 2).trim(), \"DRAFT\");\n assert.is_true(message_at(shorten, 3).includes(\"at most 12 sentences\"))\n}\n\ntest \"summarize_ebm returns the draft when num_sentences is null, else the shortened reply\" {\n let draft_only = SequenceClient { calls: 0 };\n let kept = summarize_ebm(\"IMP\", \"DESC\", null, default_graph_options(), client = draft_only);\n assert.equal(kept.text, \"REPLY-1\");\n assert.equal(kept.usage.input_tokens, 1);\n assert.equal(kept.usage.output_tokens, 1);\n\n let shortened = SequenceClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = shortened);\n assert.equal(result.text, \"REPLY-2\");\n assert.equal(result.usage.input_tokens, 2);\n assert.equal(result.usage.output_tokens, 2);\n\n // The shorten prompt holds the draft and the limit.\n let echo = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = EchoClient {});\n assert.is_true(echo.text.includes(\"at most 30 sentences\"));\n assert.is_true(echo.text.includes(\"IMP\"));\n assert.is_true(echo.text.includes(\"DESC\"))\n}\n\ntest \"describe_ebm pairs each feature name with its own description, in order\" {\n let result = describe_ebm(\n [\"age\", \"bmi\"],\n [\"G-AGE\", \"G-BMI\"],\n \"IMP\",\n 30,\n 7,\n 2,\n default_graph_options(),\n client = EchoClient {},\n );\n // The summary draft echoes the \"name: description\" lines; each description echoes its own graph.\n assert.is_true(position_of(result.text, \"age: \") >= 0);\n assert.is_true(position_of(result.text, \"age: \") < position_of(result.text, \"bmi: \"));\n assert.is_true(position_of(result.text, \"G-AGE\") < position_of(result.text, \"bmi: \"));\n assert.is_true(position_of(result.text, \"bmi: \") < position_of(result.text, \"G-BMI\"));\n assert.is_true(result.text.includes(\"at most 30 sentences\"));\n assert.is_true(result.text.includes(\"at most 7 sentence\"));\n assert.is_true(result.text.includes(\"IMP\"))\n}\n\ntest \"describe_ebm gives the same usage whatever the concurrency\" {\n let one = describe_ebm(\n [\"a\", \"b\", \"c\"],\n [\"G1\", \"G2\", \"G3\"],\n \"IMP\",\n 30,\n 7,\n 1,\n default_graph_options(),\n client = EchoClient {},\n );\n let many = describe_ebm(\n [\"a\", \"b\", \"c\"],\n [\"G1\", \"G2\", \"G3\"],\n \"IMP\",\n 30,\n 7,\n 8,\n default_graph_options(),\n client = EchoClient {},\n );\n // 3 graphs x 3 turns + draft + shorten = 11 calls at 1 input and 1 output token each.\n assert.equal(one.usage.input_tokens, 11);\n assert.equal(one.usage.output_tokens, 11);\n assert.equal(many.usage.input_tokens, 11);\n assert.equal(many.usage.output_tokens, 11)\n}\n\ntest \"describe_ebm with no graphs only summarizes\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_ebm([], [], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted);\n assert.equal(scripted.calls, 2);\n assert.equal(result.usage.input_tokens, 20)\n}\n\ntest \"describe_ebm rejects more names than graphs and more graphs than names\" {\n let scripted = ScriptedClient { calls: 0 };\n let more_graphs = describe_ebm([], [\"G\"], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted) catch (e) {\n baml.errors.InvalidArgument => null,\n };\n assert.is_true(more_graphs == null);\n assert.equal(scripted.calls, 0)\n}\n\ntest \"benchmark prompts: roles, question, reasoning and answer format\" {\n let reasoning = BenchmarkReasoning@spec(\"QUESTION\").prompt();\n assert.equal(roles_of(reasoning), \"system,user\");\n assert.is_true(\n message_at(reasoning, 0).includes(\"You are an expert statistician and data scientist.\"),\n );\n assert.is_true(message_at(reasoning, 1).includes(\"QUESTION\"));\n\n let answer = BenchmarkAnswer@spec(\"QUESTION\", \"REASONING\", \"'a' or 'b'\").prompt();\n assert.equal(roles_of(answer), \"system,user,assistant,user\");\n assert.is_true(message_at(answer, 1).includes(\"QUESTION\"));\n assert.equal(message_at(answer, 2).trim(), \"REASONING\");\n assert.is_true(message_at(answer, 3).includes(\"'a' or 'b'\"));\n assert.is_true(message_at(answer, 3).includes(\"Reply with the answer only\"))\n}\n\ntest \"answer_benchmark_question hands the reasoning to the answer turn\" {\n let sequence = SequenceClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = sequence);\n assert.equal(sequence.calls, 2);\n assert.equal(result.text, \"REPLY-2\");\n assert.equal(result.usage.input_tokens, 2);\n assert.equal(result.usage.output_tokens, 2);\n\n let echo = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = EchoClient {});\n assert.is_true(echo.text.includes(\"'a' or 'b'\"));\n assert.is_true(echo.text.includes(\"QUESTION\"));\n assert.is_true(echo.text.includes(\"Summarize your response\"))\n}\n\n// A model turn that holds `text` and reports 1 input and 1 output token.\nfunction text_turn(text: string) -> ai.ModelTurn {\n scripted_turn(text, 1, 1)\n}\n\n// Answers call n with \"Rn<<\" + its rendered prompt + \">>\". A later prompt then shows which earlier reply sits where.\n// Use it only for sequential pipelines: concurrent calls would race on the counter.\nclass TaggedEchoClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"tagged-echo\"\n }\n\n function requested_model(self) -> string {\n \"tagged-echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://tagged-echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n let prompt = input.prompt(ai.internal.build_output_format(input.output_type));\n text_turn(`R${self.calls}<<${prompt.text()}>>`)\n }\n }\n}\n\n// Answers \"R\" to graph prompts and echoes the summary prompts, so the graph descriptions stay predictable.\nclass SummaryEchoClient {\n implements ai.Client {\n function provider(self) -> string {\n \"summary-echo\"\n }\n\n function requested_model(self) -> string {\n \"summary-echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://summary-echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n let text = input.prompt(ai.internal.build_output_format(input.output_type)).text();\n if (text.includes(\"overall summary of a Generalized Additive Model\")) {\n return text_turn(text);\n };\n text_turn(\"R\")\n }\n }\n}\n\ntest \"describe_graph_cot puts the first reply in the second and third turns and the second reply in the third\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = tagged);\n assert.is_true(result.text.starts_with(\"R3<<\"));\n // The third prompt holds R1 (with the first prompt inside it) before R2, and R2 holds R1 too.\n let r1 = position_of(result.text, \"R1<<\");\n let r2 = position_of(result.text, \"R2<<\");\n assert.is_true(r1 > 0);\n assert.is_true(r1 < r2);\n assert.is_true(position_of(result.text.slice(r2, result.text.length()), \"R1<<\") > 0)\n}\n\ntest \"describe_ebm joins the per-feature descriptions with a blank line\" {\n let result = describe_ebm(\n [\"age\", \"bmi\"],\n [\"G1\", \"G2\"],\n \"IMP\",\n 30,\n 7,\n 2,\n default_graph_options(),\n client = SummaryEchoClient {},\n );\n assert.is_true(result.text.includes(\"age: R\\n\\nbmi: R\"))\n}\n\ntest \"answer_benchmark_question puts the reasoning reply in the answer prompt\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = tagged);\n assert.is_true(result.text.starts_with(\"R2<<\"));\n assert.is_true(result.text.includes(\"R1<<\"));\n assert.is_true(\n position_of(result.text, \"R1<<\") < position_of(result.text, \"Summarize your response\"),\n )\n}\n\ntest \"summarize_ebm puts the draft reply in the shorten prompt\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = tagged);\n assert.is_true(result.text.starts_with(\"R2<<\"));\n assert.is_true(position_of(result.text, \"R1<<\") > 0);\n assert.is_true(\n position_of(result.text, \"R1<<\") < position_of(result.text, \"at most 30 sentences\"),\n )\n}\n\ntest \"omitted_parts_msg names each omitted part and says why, and is empty without one\" {\n assert.equal(omitted_parts_msg([], []), \"\");\n let both = omitted_parts_msg([\"age & income\", \"age & city\"], [\"zip\"]);\n assert.is_true(both.includes(\"<omitted_parts>\") && both.includes(\"</omitted_parts>\"));\n assert.is_true(\n both.includes(\n \"interaction terms, and no graph description covers them: age & income, age & city.\",\n ),\n );\n assert.is_true(\n both.includes(\"no main effect in the model, and no graph description covers them: zip.\"),\n );\n assert.is_true(both.includes(\"Say in one sentence that your answer leaves these parts out\"));\n assert.is_true(both.includes(\"does not cover the whole model\"));\n let terms = omitted_parts_msg([\"age & income\"], []);\n assert.is_true(terms.includes(\"age & income\") && !terms.includes(\"no main effect\"));\n let features = omitted_parts_msg([], [\"zip\"]);\n assert.is_true(features.includes(\"zip\") && !features.includes(\"interaction terms\"))\n}\n\ntest \"the summary prompts name the omitted parts, and a model without them gets no such text\" {\n // Contract 4: a description that leaves out a model part says so in its text, not only in a log.\n let plain = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"\");\n assert.is_true(!plain.includes(\"omitted_parts\"));\n let msg = summarize_ebm_msg(\n \"IMPORTANCES\",\n \"DESCRIPTIONS\",\n \"\",\n omitted_terms = [\"age & income\"],\n omitted_features = [\"zip\"],\n );\n assert.is_true(position_of(msg, \"DESCRIPTIONS\") < position_of(msg, \"<omitted_parts>\"));\n assert.is_true(position_of(msg, \"</omitted_parts>\") < position_of(msg, \"<instructions>\"));\n\n let result = summarize_ebm(\n \"IMPORTANCES\",\n \"DESCRIPTIONS\",\n 3,\n default_graph_options(),\n client = EchoClient {},\n omitted_terms = [\"age & income\"],\n omitted_features = [\"zip\"],\n );\n // the echo of the shorten turn holds the draft turn too, so both turns got the block\n assert.is_true(result.text.includes(\"age & income\") && result.text.includes(\"zip\"));\n assert.is_true(\n result.text.includes(\"keep the sentence that names the parts that the summary leaves out\"),\n );\n let draft = summarize_ebm(\n \"IMPORTANCES\",\n \"DESCRIPTIONS\",\n null,\n default_graph_options(),\n client = EchoClient {},\n omitted_terms = [\"age & income\"],\n );\n assert.is_true(draft.text.includes(\"age & income\") && !draft.text.includes(\"no main effect\"))\n}\n\ntest \"describe_graph_cot names the omitted interaction terms in its final turn only\" {\n assert.is_true(!final_description_request(3).includes(\"omitted_parts\"));\n assert.is_true(\n final_description_request(3, omitted_terms = [\"age & income\"]).includes(\"age & income\"),\n );\n let opts = default_graph_options();\n let first = GraphFirstPass@spec(\"GRAPH\", opts).prompt().text();\n let final = GraphFinalDescription@spec(\"GRAPH\", \"FIRST\", \"SURPRISES\", 3, opts, omitted_terms = [\"age & income\"])\n .prompt()\n .text();\n assert.is_true(!first.includes(\"age & income\"));\n assert.is_true(final.includes(\"age & income\") && final.includes(\"at most 3 sentences\"));\n let result = describe_graph_cot(\"GRAPH\", 3, opts, client = EchoClient {}, omitted_terms = [\"age & income\"]);\n assert.is_true(result.text.includes(\"age & income\"))\n}\n\ntest \"describe_ebm hands the omitted parts to the summary\" {\n let result = describe_ebm(\n [\"age\"],\n [\"GRAPH\"],\n \"IMPORTANCES\",\n 3,\n 2,\n 1,\n default_graph_options(),\n client = EchoClient {},\n omitted_terms = [\"age & income\"],\n omitted_features = [\"zip\"],\n );\n assert.is_true(result.text.includes(\"<omitted_parts>\"));\n assert.is_true(result.text.includes(\"age & income\") && result.text.includes(\"zip\"))\n}\n",