ebaml 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ebaml-0.1.0 → ebaml-0.2.0}/PKG-INFO +28 -10
- {ebaml-0.1.0 → ebaml-0.2.0}/README.md +27 -9
- {ebaml-0.1.0 → ebaml-0.2.0}/pyproject.toml +2 -2
- {ebaml-0.1.0 → ebaml-0.2.0}/pyproject.toml.orig +2 -2
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/.baml-generator-output.json +4 -4
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/__init__.py +6 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/__init__.pyi +6 -0
- ebaml-0.2.0/src/baml_sdk/_baml_sources.py +31 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/_inlinedbaml.py +1 -1
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/__init__.py +12 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/core/custom_types.py +16 -3
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/graphs/ebm_graph.py +223 -48
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/graphs/graph_text.py +85 -19
- ebaml-0.2.0/src/ebaml/graphs/model_kinds.py +227 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/llm/descriptors.py +99 -27
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/llm/options.py +6 -1
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/llm/provider_errors.py +1 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/llm/retry.py +8 -1
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/local/fact_text.py +28 -27
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/local/facts.py +150 -45
- ebaml-0.1.0/src/baml_sdk/_baml_sources.py +0 -31
- {ebaml-0.1.0 → ebaml-0.2.0}/LICENSE +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/.baml-generator-output-root +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/.gitignore +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/_function_registry.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/_invocation_types.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/_invocation_types.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/_typemap.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/clients/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/clients/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/content/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/content/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/errors/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/errors/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/events/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/events/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/mcp/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/mcp/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/stream/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/stream/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/tools/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/tools/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/wire/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/ai/wire/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/crypto/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/crypto/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/csv/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/csv/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/env/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/env/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/errors/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/errors/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/fs/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/fs/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/future/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/future/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/glob/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/glob/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/hash/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/hash/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/host/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/host/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/http/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/http/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/io/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/io/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/iter/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/iter/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/json/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/json/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/media/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/media/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/net/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/net/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/ops/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/ops/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/panics/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/panics/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/random/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/random/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/regex/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/regex/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/sap/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/sap/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/spawn/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/spawn/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/sys/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/sys/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/time/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/time/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/toml/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/toml/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/ws/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/ws/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/yaml/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/baml/yaml/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/experimental/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/experimental/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/py.typed +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/class_/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/class_/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/enum/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/enum/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/errors/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/errors/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/function/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/function/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/interface/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/reflect/interface/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/anthropic/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/anthropic/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/anthropic/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/anthropic/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/assert_/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/assert_/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/aws/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/aws/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/aws/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/aws/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/claude_code/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/claude_code/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/claude_code/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/claude_code/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/google/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/google/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/google/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/google/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/openai/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/openai/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/openai/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/openai/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/openai/internal/decisions/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/openai/internal/decisions/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/testing/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/testing/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/trace/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/trace/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/typesafeai/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/typesafeai/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/typesafeai/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/typesafeai/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/vercel/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/vercel/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/vercel/internal/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/baml_sdk/vendor/vercel/internal/__init__.pyi +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/benchmark.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/core/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/core/exceptions.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/graphs/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/llm/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/llm/sdk_setup.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/local/__init__.py +0 -0
- {ebaml-0.1.0 → ebaml-0.2.0}/src/ebaml/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ebaml
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: BAML for Explainable Boosting Machines: talk to your EBM.
|
|
5
5
|
Keywords: baml,llm,explainable-ai,explainable-boosting-machine,interpretml
|
|
6
6
|
Author: Johnny Godoy
|
|
@@ -45,13 +45,13 @@ LLM calls go through [OpenRouter](https://openrouter.ai)'s free router (`openrou
|
|
|
45
45
|
<!-- --8<-- [start:supported-models] -->
|
|
46
46
|
| Feature | Binary classifier | Regressor | Multiclass classifier | Interactions |
|
|
47
47
|
| --- | :---: | :---: | :---: | :---: |
|
|
48
|
-
| Graph descriptions (`describe_graph`) | ✅ | ✅ |
|
|
49
|
-
| Model summary (`describe_ebm`) | ✅ | ✅ |
|
|
50
|
-
| Graph text (`EBMGraph`) | ✅ | ✅ |
|
|
48
|
+
| Graph descriptions (`describe_graph`) | ✅ | ✅ | ✅ | ⚠️ |
|
|
49
|
+
| Model summary (`describe_ebm`) | ✅ | ✅ | ✅ | ⚠️ |
|
|
50
|
+
| Graph text (`EBMGraph`) | ✅ | ✅ | ✅ | ⚠️ |
|
|
51
51
|
| Local explanations (`explain_instance`) | ✅ | ❌ | ❌ | ❌ |
|
|
52
52
|
|
|
53
|
-
✅ supported. ⚠️ partly supported: the main effects are described, and the interaction terms are skipped
|
|
54
|
-
warning. ❌ not supported yet. Full support is on the
|
|
53
|
+
✅ supported. ⚠️ partly supported: the main effects are described, and the interaction terms are skipped.
|
|
54
|
+
`describe_graph` and `describe_ebm` log an `interactions_skipped` warning that names them. ❌ not supported yet. Full support is on the
|
|
55
55
|
[roadmap](https://github.com/johnny-godoy/EBaMl/blob/main/ROADMAP.md).
|
|
56
56
|
<!-- --8<-- [end:supported-models] -->
|
|
57
57
|
## Setup
|
|
@@ -183,6 +183,24 @@ Three keyword-only arguments set the options. `graph=GraphOptions(...)` sets the
|
|
|
183
183
|
(`expert_description`, `graph_description`, `dataset_description` and `task_description`). `model=` selects the LLM, see
|
|
184
184
|
"Choosing the model". The classes are in `ebaml.llm.options`. Token usage reported by the provider is logged at `INFO` on the `ebaml.llm.descriptors` logger.
|
|
185
185
|
|
|
186
|
+
A multiclass classifier has one graph series per class. `GraphOptions(classes=[...])` selects the classes to describe,
|
|
187
|
+
because the text of a model with many classes is long. In every bin, the scores have the same mean over all the classes
|
|
188
|
+
of the model. Only the differences between the classes change the predicted probabilities.
|
|
189
|
+
`notebooks/multiclass.py` shows each option on the iris data.
|
|
190
|
+
|
|
191
|
+
<!-- invisible-code-block: python
|
|
192
|
+
import numpy as np
|
|
193
|
+
|
|
194
|
+
y_multiclass = np.select([X["age"] < 30, X["age"] < 60], ["young", "middle"], "old")
|
|
195
|
+
-->
|
|
196
|
+
|
|
197
|
+
```python
|
|
198
|
+
from ebaml.llm.options import GraphOptions
|
|
199
|
+
|
|
200
|
+
multiclass = ExplainableBoostingClassifier(interactions=0).fit(X, y_multiclass)
|
|
201
|
+
print(describe_graph(multiclass, feature_index=0, graph=GraphOptions(classes=["young", "old"])))
|
|
202
|
+
```
|
|
203
|
+
|
|
186
204
|
### Explaining one prediction
|
|
187
205
|
|
|
188
206
|
```python
|
|
@@ -215,10 +233,10 @@ of `ebaml` apart from the errors of other code. Each error for a bad value also
|
|
|
215
233
|
| `InvalidOptionError` | yes | An option is not one of its valid values, for example an unknown `Normalization`. |
|
|
216
234
|
| `OptionRangeError` | yes | A number option is outside its range, for example `confidence_level=1`. |
|
|
217
235
|
| `InvalidInstanceError` | yes | The instance to explain lacks a value, or has a category that the EBM did not see. |
|
|
218
|
-
| `UnsupportedModelError` | yes | A local explanation gets an EBM with interactions, or one that is not a binary classifier. |
|
|
236
|
+
| `UnsupportedModelError` | yes | A local explanation gets an EBM with interactions, or one that is not a binary classifier. A graph gets `classes`, and the model has one output. `model_kind` gets an EBM that is not fitted. |
|
|
219
237
|
| `GraphTextError` | yes | `EBMGraph.from_text` cannot read the text of a graph. |
|
|
220
238
|
| `EBMGraphError` | no | The base class of the errors of a graph. |
|
|
221
|
-
| `EBMGraphShapeError` | yes | The bins of a graph do not fit its scores, or a boolean format gets more than two bins. |
|
|
239
|
+
| `EBMGraphShapeError` | yes | The bins of a graph do not fit its scores, the scores and the standard deviations have different shapes, or a boolean format gets more than two bins. |
|
|
222
240
|
| `EBMGraphTokenLimitError` | no | The text of a graph is over the token budget, and its format cannot be simplified. |
|
|
223
241
|
| `EBMGraphSimplificationError` | no | The text of a graph is still over the token budget at the largest simplification. |
|
|
224
242
|
| `SDKUnavailableError` | no | `baml_sdk` is missing, and `ebaml` cannot run `baml generate`. The error also derives from `ImportError`. |
|
|
@@ -430,7 +448,7 @@ uv run pytest # Python unit tests, fails under 100% coverage
|
|
|
430
448
|
uv run pytest README.md docs src --no-cov # the code examples of the docs and the docstrings
|
|
431
449
|
prek run --all-files # every hook: formatters, ruff, vulture, pyrefly, baml fmt, ...
|
|
432
450
|
uv run python scripts/mutation.py # mutation testing of all modules, fails if any mutant survives
|
|
433
|
-
uv run python scripts/mutation.py --base origin/main # the same, but only for
|
|
451
|
+
uv run python scripts/mutation.py --base origin/main # the same, but only for the lines changed since main
|
|
434
452
|
uv run python scripts/mutation.py --module src/ebaml/local/facts.py # the same, but only for one module
|
|
435
453
|
uv run python -m scripts.check_ignores # every lint/type/test workaround needs a reason written next to it
|
|
436
454
|
uv run python -m scripts.compile_ignores # all workarounds and their reasons as one Markdown report to audit
|
|
@@ -439,7 +457,7 @@ uv run --group docs python -m scripts.build_docs serve # preview the docs (exp
|
|
|
439
457
|
```
|
|
440
458
|
|
|
441
459
|
CI (`.github/workflows/ci.yml`) runs prek, secret and security scans (trufflehog, semgrep, trivy), `baml check` and
|
|
442
|
-
`baml test`, pytest with 100% coverage, the code examples of the documentation, and mutation testing of the changed
|
|
460
|
+
`baml test`, pytest with 100% coverage, the code examples of the documentation, and, in pull requests, mutation testing of the changed lines with a 100% score. A weekly run tests all modules, one job for each module. Lint and type workarounds
|
|
443
461
|
(`# noqa`, `# type: ignore`, skipped tests, ...) must carry a written reason next to them.
|
|
444
462
|
|
|
445
463
|
The `python` code blocks of `README.md` and `docs/*.md` run in order, and the blocks of one file share their variables.
|
|
@@ -17,13 +17,13 @@ LLM calls go through [OpenRouter](https://openrouter.ai)'s free router (`openrou
|
|
|
17
17
|
<!-- --8<-- [start:supported-models] -->
|
|
18
18
|
| Feature | Binary classifier | Regressor | Multiclass classifier | Interactions |
|
|
19
19
|
| --- | :---: | :---: | :---: | :---: |
|
|
20
|
-
| Graph descriptions (`describe_graph`) | ✅ | ✅ |
|
|
21
|
-
| Model summary (`describe_ebm`) | ✅ | ✅ |
|
|
22
|
-
| Graph text (`EBMGraph`) | ✅ | ✅ |
|
|
20
|
+
| Graph descriptions (`describe_graph`) | ✅ | ✅ | ✅ | ⚠️ |
|
|
21
|
+
| Model summary (`describe_ebm`) | ✅ | ✅ | ✅ | ⚠️ |
|
|
22
|
+
| Graph text (`EBMGraph`) | ✅ | ✅ | ✅ | ⚠️ |
|
|
23
23
|
| Local explanations (`explain_instance`) | ✅ | ❌ | ❌ | ❌ |
|
|
24
24
|
|
|
25
|
-
✅ supported. ⚠️ partly supported: the main effects are described, and the interaction terms are skipped
|
|
26
|
-
warning. ❌ not supported yet. Full support is on the
|
|
25
|
+
✅ supported. ⚠️ partly supported: the main effects are described, and the interaction terms are skipped.
|
|
26
|
+
`describe_graph` and `describe_ebm` log an `interactions_skipped` warning that names them. ❌ not supported yet. Full support is on the
|
|
27
27
|
[roadmap](https://github.com/johnny-godoy/EBaMl/blob/main/ROADMAP.md).
|
|
28
28
|
<!-- --8<-- [end:supported-models] -->
|
|
29
29
|
## Setup
|
|
@@ -155,6 +155,24 @@ Three keyword-only arguments set the options. `graph=GraphOptions(...)` sets the
|
|
|
155
155
|
(`expert_description`, `graph_description`, `dataset_description` and `task_description`). `model=` selects the LLM, see
|
|
156
156
|
"Choosing the model". The classes are in `ebaml.llm.options`. Token usage reported by the provider is logged at `INFO` on the `ebaml.llm.descriptors` logger.
|
|
157
157
|
|
|
158
|
+
A multiclass classifier has one graph series per class. `GraphOptions(classes=[...])` selects the classes to describe,
|
|
159
|
+
because the text of a model with many classes is long. In every bin, the scores have the same mean over all the classes
|
|
160
|
+
of the model. Only the differences between the classes change the predicted probabilities.
|
|
161
|
+
`notebooks/multiclass.py` shows each option on the iris data.
|
|
162
|
+
|
|
163
|
+
<!-- invisible-code-block: python
|
|
164
|
+
import numpy as np
|
|
165
|
+
|
|
166
|
+
y_multiclass = np.select([X["age"] < 30, X["age"] < 60], ["young", "middle"], "old")
|
|
167
|
+
-->
|
|
168
|
+
|
|
169
|
+
```python
|
|
170
|
+
from ebaml.llm.options import GraphOptions
|
|
171
|
+
|
|
172
|
+
multiclass = ExplainableBoostingClassifier(interactions=0).fit(X, y_multiclass)
|
|
173
|
+
print(describe_graph(multiclass, feature_index=0, graph=GraphOptions(classes=["young", "old"])))
|
|
174
|
+
```
|
|
175
|
+
|
|
158
176
|
### Explaining one prediction
|
|
159
177
|
|
|
160
178
|
```python
|
|
@@ -187,10 +205,10 @@ of `ebaml` apart from the errors of other code. Each error for a bad value also
|
|
|
187
205
|
| `InvalidOptionError` | yes | An option is not one of its valid values, for example an unknown `Normalization`. |
|
|
188
206
|
| `OptionRangeError` | yes | A number option is outside its range, for example `confidence_level=1`. |
|
|
189
207
|
| `InvalidInstanceError` | yes | The instance to explain lacks a value, or has a category that the EBM did not see. |
|
|
190
|
-
| `UnsupportedModelError` | yes | A local explanation gets an EBM with interactions, or one that is not a binary classifier. |
|
|
208
|
+
| `UnsupportedModelError` | yes | A local explanation gets an EBM with interactions, or one that is not a binary classifier. A graph gets `classes`, and the model has one output. `model_kind` gets an EBM that is not fitted. |
|
|
191
209
|
| `GraphTextError` | yes | `EBMGraph.from_text` cannot read the text of a graph. |
|
|
192
210
|
| `EBMGraphError` | no | The base class of the errors of a graph. |
|
|
193
|
-
| `EBMGraphShapeError` | yes | The bins of a graph do not fit its scores, or a boolean format gets more than two bins. |
|
|
211
|
+
| `EBMGraphShapeError` | yes | The bins of a graph do not fit its scores, the scores and the standard deviations have different shapes, or a boolean format gets more than two bins. |
|
|
194
212
|
| `EBMGraphTokenLimitError` | no | The text of a graph is over the token budget, and its format cannot be simplified. |
|
|
195
213
|
| `EBMGraphSimplificationError` | no | The text of a graph is still over the token budget at the largest simplification. |
|
|
196
214
|
| `SDKUnavailableError` | no | `baml_sdk` is missing, and `ebaml` cannot run `baml generate`. The error also derives from `ImportError`. |
|
|
@@ -402,7 +420,7 @@ uv run pytest # Python unit tests, fails under 100% coverage
|
|
|
402
420
|
uv run pytest README.md docs src --no-cov # the code examples of the docs and the docstrings
|
|
403
421
|
prek run --all-files # every hook: formatters, ruff, vulture, pyrefly, baml fmt, ...
|
|
404
422
|
uv run python scripts/mutation.py # mutation testing of all modules, fails if any mutant survives
|
|
405
|
-
uv run python scripts/mutation.py --base origin/main # the same, but only for
|
|
423
|
+
uv run python scripts/mutation.py --base origin/main # the same, but only for the lines changed since main
|
|
406
424
|
uv run python scripts/mutation.py --module src/ebaml/local/facts.py # the same, but only for one module
|
|
407
425
|
uv run python -m scripts.check_ignores # every lint/type/test workaround needs a reason written next to it
|
|
408
426
|
uv run python -m scripts.compile_ignores # all workarounds and their reasons as one Markdown report to audit
|
|
@@ -411,7 +429,7 @@ uv run --group docs python -m scripts.build_docs serve # preview the docs (exp
|
|
|
411
429
|
```
|
|
412
430
|
|
|
413
431
|
CI (`.github/workflows/ci.yml`) runs prek, secret and security scans (trufflehog, semgrep, trivy), `baml check` and
|
|
414
|
-
`baml test`, pytest with 100% coverage, the code examples of the documentation, and mutation testing of the changed
|
|
432
|
+
`baml test`, pytest with 100% coverage, the code examples of the documentation, and, in pull requests, mutation testing of the changed lines with a 100% score. A weekly run tests all modules, one job for each module. Lint and type workarounds
|
|
415
433
|
(`# noqa`, `# type: ignore`, skipped tests, ...) must carry a written reason next to them.
|
|
416
434
|
|
|
417
435
|
The `python` code blocks of `README.md` and `docs/*.md` run in order, and the blocks of one file share their variables.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ebaml"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "BAML for Explainable Boosting Machines: talk to your EBM."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -239,7 +239,7 @@ categories = [
|
|
|
239
239
|
"Security",
|
|
240
240
|
]
|
|
241
241
|
md_header_level = "2"
|
|
242
|
-
version = "
|
|
242
|
+
version = "literal: pyproject.toml: project.version"
|
|
243
243
|
entry_title_template = "[{{ version }}] - {{ date.strftime('%Y-%m-%d') }}"
|
|
244
244
|
|
|
245
245
|
[tool.commitizen]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ebaml"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.2.0"
|
|
4
4
|
description = "BAML for Explainable Boosting Machines: talk to your EBM."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -271,7 +271,7 @@ format = "md"
|
|
|
271
271
|
fragment_directory = "changelog.d"
|
|
272
272
|
categories = ["Added", "Changed", "Deprecated", "Removed", "Fixed", "Security"]
|
|
273
273
|
md_header_level = "2"
|
|
274
|
-
version = "
|
|
274
|
+
version = "literal: pyproject.toml: project.version"
|
|
275
275
|
entry_title_template = "[{{ version }}] - {{ date.strftime('%Y-%m-%d') }}"
|
|
276
276
|
|
|
277
277
|
# commitizen only checks commit messages and PR titles here. scriv and scripts/release.py manage the changelog and the
|
|
@@ -7,15 +7,15 @@
|
|
|
7
7
|
},
|
|
8
8
|
{
|
|
9
9
|
"relative_path": "__init__.py",
|
|
10
|
-
"sha256": "
|
|
10
|
+
"sha256": "409ca63a082ceaa731e7d616981a20014019d488eeb9fdb3ffe6c6210104deb9"
|
|
11
11
|
},
|
|
12
12
|
{
|
|
13
13
|
"relative_path": "__init__.pyi",
|
|
14
|
-
"sha256": "
|
|
14
|
+
"sha256": "d91c98a9aa7242817f0efa01583c7e21ac2e2643110f8793c722495df49f0503"
|
|
15
15
|
},
|
|
16
16
|
{
|
|
17
17
|
"relative_path": "_baml_sources.py",
|
|
18
|
-
"sha256": "
|
|
18
|
+
"sha256": "7d7a5950cfe53e2daa682060b8462204e9aa931966dae9a3722b404ab64f55c3"
|
|
19
19
|
},
|
|
20
20
|
{
|
|
21
21
|
"relative_path": "_function_registry.py",
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
},
|
|
24
24
|
{
|
|
25
25
|
"relative_path": "_inlinedbaml.py",
|
|
26
|
-
"sha256": "
|
|
26
|
+
"sha256": "1d0e15cb9743010c198cfdd8002d959b3ba2e775d3f4e88af7fd66e14820d115"
|
|
27
27
|
},
|
|
28
28
|
{
|
|
29
29
|
"relative_path": "_invocation_types.py",
|
|
@@ -333,6 +333,10 @@ graph_system_msg = _define_function("user.graph_system_msg", "sync", ["ex
|
|
|
333
333
|
graph_system_msg_async = _define_function("user.graph_system_msg", "async", ["expert_description"], binding_name="graph_system_msg_async", binding_qualname="graph_system_msg_async", binding_module=__name__)
|
|
334
334
|
|
|
335
335
|
|
|
336
|
+
multiclass_format = _define_function("user.multiclass_format", "sync", ["graph"], binding_name="multiclass_format", binding_qualname="multiclass_format", binding_module=__name__)
|
|
337
|
+
multiclass_format_async = _define_function("user.multiclass_format", "async", ["graph"], binding_name="multiclass_format_async", binding_qualname="multiclass_format_async", binding_module=__name__)
|
|
338
|
+
|
|
339
|
+
|
|
336
340
|
describe_graph_msg = _define_function("user.describe_graph_msg", "sync", ["graph", "opts"], binding_name="describe_graph_msg", binding_qualname="describe_graph_msg", binding_module=__name__)
|
|
337
341
|
describe_graph_msg_async = _define_function("user.describe_graph_msg", "async", ["graph", "opts"], binding_name="describe_graph_msg_async", binding_qualname="describe_graph_msg_async", binding_module=__name__)
|
|
338
342
|
|
|
@@ -728,6 +732,8 @@ __all__ = [
|
|
|
728
732
|
"default_graph_options_async",
|
|
729
733
|
"graph_system_msg",
|
|
730
734
|
"graph_system_msg_async",
|
|
735
|
+
"multiclass_format",
|
|
736
|
+
"multiclass_format_async",
|
|
731
737
|
"describe_graph_msg",
|
|
732
738
|
"describe_graph_msg_async",
|
|
733
739
|
"summarize_ebm_system_msg",
|
|
@@ -298,6 +298,10 @@ def graph_system_msg(expert_description: str, *, _baml: _BamlOptions | None = No
|
|
|
298
298
|
async def graph_system_msg_async(expert_description: str, *, _baml: _BamlOptions | None = None) -> str: ...
|
|
299
299
|
|
|
300
300
|
|
|
301
|
+
def multiclass_format(graph: str, *, _baml: _BamlOptions | None = None) -> str: ...
|
|
302
|
+
async def multiclass_format_async(graph: str, *, _baml: _BamlOptions | None = None) -> str: ...
|
|
303
|
+
|
|
304
|
+
|
|
301
305
|
def describe_graph_msg(graph: str, opts: GraphPromptOptions, *, _baml: _BamlOptions | None = None) -> str: ...
|
|
302
306
|
async def describe_graph_msg_async(graph: str, opts: GraphPromptOptions, *, _baml: _BamlOptions | None = None) -> str: ...
|
|
303
307
|
|
|
@@ -655,6 +659,8 @@ __all__ = [
|
|
|
655
659
|
"default_graph_options_async",
|
|
656
660
|
"graph_system_msg",
|
|
657
661
|
"graph_system_msg_async",
|
|
662
|
+
"multiclass_format",
|
|
663
|
+
"multiclass_format_async",
|
|
658
664
|
"describe_graph_msg",
|
|
659
665
|
"describe_graph_msg_async",
|
|
660
666
|
"summarize_ebm_system_msg",
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# ruff: noqa
|
|
2
|
+
# flake8: noqa
|
|
3
|
+
# pylint: skip-file
|
|
4
|
+
# mypy: ignore-errors
|
|
5
|
+
# pyright: reportPrivateUsage=false, reportUnknownArgumentType=false
|
|
6
|
+
# fmt: off
|
|
7
|
+
# isort: skip_file
|
|
8
|
+
|
|
9
|
+
# ----------------------------------------------------------------------------
|
|
10
|
+
#
|
|
11
|
+
# Welcome to Baml! To use this generated code, run one of the following:
|
|
12
|
+
#
|
|
13
|
+
# $ uv add baml-bridge
|
|
14
|
+
# $ pip install baml-bridge
|
|
15
|
+
# $ conda run python -m pip install baml-bridge
|
|
16
|
+
#
|
|
17
|
+
# ----------------------------------------------------------------------------
|
|
18
|
+
|
|
19
|
+
# This file was generated by BAML: please do not edit it. Instead, edit the
|
|
20
|
+
# BAML files and re-generate this code using `baml generate`
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
FILES: dict[str, str] = {
|
|
25
|
+
"local_explanations.baml": "// Local explanations: Python computes the facts of one prediction deterministically (ebaml.local), BAML turns each\n// kind of fact into words. The three kinds are the explainability queries of Barceló et al., \"Model Interpretability\n// through the Lens of Computational Complexity\" (arXiv:2010.12265): minimum change required, sufficient reason and\n// count completions. The global explanation of the model (describe_ebm) is prepended to every prompt.\n\n// Facts of one prediction, one JSON text per kind of explanation.\nclass LocalFactTexts {\n prediction: string,\n counterfactual: string,\n sufficient_reason: string,\n completions: string,\n}\n\nclass LocalExplanationText {\n counterfactual: string,\n sufficient_reason: string,\n completions: string,\n usage: TokenUsage,\n}\n\nfunction local_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You explain individual predictions of a Generalized Additive Model (GAM). You receive an explanation of the whole model and facts about one prediction that were computed exactly from the model. State only effects, values and probabilities that appear in these facts, because the reader acts on your words and an invented number would mislead. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// The audience decides the vocabulary. The facts of a \"layperson\" carry no model numbers, but the instruction repeats it.\nfunction audience_guidance(audience: \"layperson\" | \"data_scientist\") -> string {\n if (audience == \"data_scientist\") {\n return \"The reader is a data scientist, so be precise. Quote the scores (log-odds contributions), probabilities and confidence intervals from the facts. Say how uncertain a statement is when the facts show wide confidence intervals.\";\n };\n \"The reader is not technical and knows no statistics, so write in everyday words. Express strength and likelihood in plain words such as 'strongly', 'slightly', 'very likely' or 'about as likely as not'. Mention only values of the person's own inputs (for example their age), because model numbers (scores, log-odds, probabilities, percentages, intervals, standard deviations) mean nothing to this reader.\"\n}\n\nfunction local_user_msg(\n global_explanation: string,\n prediction: string,\n question: string,\n facts: string,\n audience: \"layperson\" | \"data_scientist\",\n dataset_description: string,\n) -> string {\n let msg = `\n <global_explanation>\n ${global_explanation}\n </global_explanation>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + `\n <prediction>\n ${prediction}\n </prediction>\n\n <facts>\n ${facts}\n </facts>\n\n <audience>\n ${audience_guidance(audience)}\n </audience>\n\n <question>\n ${question}\n </question>\n `\n}\n\nfunction counterfactual_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, what would have to change for the prediction to be different.\n The facts list the smallest set of input changes that flips the prediction (a minimum change required), and whether the flip survives the model's uncertainty.\n This set is exact, not a guess: no smaller set of changes flips the prediction, because the model adds up one independent effect per input.\n Only realistic alternative values were considered (rare extreme values are left out), and the flip holds even if the model's effects are at the unfavourable edge of their confidence interval.\n Suggest only the changes that the facts list, because they are the only ones that the model supports.\n If no change flips the prediction, say that the prediction cannot be changed by altering the inputs.\n `\n}\n\nfunction sufficient_reason_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, which inputs are enough to settle the prediction.\n The facts list a smallest set of inputs (a sufficient reason) that fixes the prediction whatever the other inputs are. State clearly that the remaining inputs cannot change the outcome.\n This is exact: no smaller set of inputs guarantees the outcome, because the model adds up one independent effect per input.\n The guarantee covers every realistic value of the other inputs (rare extreme values are left out) and holds even when the model's effects are at the unfavourable edge of their confidence interval.\n Present it as a statement about this model, not about the real world.\n If there is no such set, say that every input matters.\n `\n}\n\nfunction completions_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, how the outcome gets more certain as the inputs become known.\n The facts give the share of people who would get the same prediction when only the first inputs are known and the rest are unknown (count completions).\n Describe how that share grows and which inputs make the biggest difference.\n Unlike the other two explanations, these shares are estimates from a large random sample that treats the unknown inputs as independent and distributed as in the training data. Describe them as approximate, because they are not guarantees.\n `\n}\n\nfunction LocalCounterfactual(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, counterfactual_question(num_sentences), facts.counterfactual, audience, opts.dataset_description)}\n `\n}\n\nfunction LocalSufficientReason(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, sufficient_reason_question(num_sentences), facts.sufficient_reason, audience, opts.dataset_description)}\n `\n}\n\nfunction LocalCompletions(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, completions_question(num_sentences), facts.completions, audience, opts.dataset_description)}\n `\n}\n\n// Entry point called from Python: the three explanations are independent, so they run concurrently.\nfunction explain_local(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> LocalExplanationText {\n let selected = select_client(client, model, short_defaults());\n let counterfactual = spawn {\n run_counted(\n LocalCounterfactual@spec(global_explanation, facts, audience, num_sentences, opts),\n client = selected,\n )\n };\n let sufficient = spawn {\n run_counted(\n LocalSufficientReason@spec(global_explanation, facts, audience, num_sentences, opts),\n client = selected,\n )\n };\n let completions = spawn {\n run_counted(\n LocalCompletions@spec(global_explanation, facts, audience, num_sentences, opts),\n client = selected,\n )\n };\n let done = await baml.future.all([counterfactual, sufficient, completions]);\n LocalExplanationText {\n counterfactual: done[0].text,\n sufficient_reason: done[1].text,\n completions: done[2].text,\n usage: add_usage(add_usage(done[0].usage, done[1].usage), done[2].usage),\n }\n}\n",
|
|
26
|
+
"model_spec.baml": "// A model chosen at run time. Python sends a `ModelSpec` (plain data) and BAML builds the client, because a client\n// object cannot cross the Python/BAML boundary.\n\nclass ModelSpec {\n provider: string,\n model: string,\n api_key_env: string?,\n base_url: string?,\n temperature: float?,\n max_output_tokens: int?,\n headers: map<string, string>?,\n}\n\n// The settings of one call: used for each field that the `ModelSpec` leaves null.\nclass CallDefaults {\n temperature: float,\n max_output_tokens: int,\n}\n\n// These three must equal the `temperature` and `max_tokens` of `RouterBase`, `RouterLongBase` and\n// `RouterBenchmarkBase` in talk_to_ebm.baml.\nfunction short_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens: 6000 }\n}\n\n// The first turn of each conversation allows a longer answer.\nfunction long_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens: 8000 }\n}\n\nfunction benchmark_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.0, max_output_tokens: 8000 }\n}\n\n// A field that the user set wins. Otherwise the default of the call applies.\nfunction resolve_settings(spec: ModelSpec, defaults: CallDefaults) -> CallDefaults {\n CallDefaults {\n temperature: spec.temperature ?? defaults.temperature,\n max_output_tokens: spec.max_output_tokens ?? defaults.max_output_tokens,\n }\n}\n\n// The key is read from the environment when the request is sent. Null selects the default variable of the provider.\nfunction credential(name: string?) -> ai.Credential? {\n if (name == null) {\n return null;\n };\n baml.env.ref(name)\n}\n\nfunction build_openai(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.ChatClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_openai_generic(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.GenericClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// Azure serves deployments under names that the owner chose, so `model` names the deployment. `base_url` names it too\n// and wins. Without it, the deployment is built on the endpoint in `AZURE_OPENAI_ENDPOINT`.\nfunction build_azure(spec: ModelSpec, settings: CallDefaults) -> openai.AzureClient {\n openai.AzureClient.new(\n model = spec.model,\n deployment_id = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_openrouter(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.OpenRouterClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_ollama(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.OllamaClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_anthropic(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n anthropic.Client.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// Gemini names its output limit `max_output_tokens`.\nfunction build_google(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n google.GeminiClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_output_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_vertex(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n google.VertexClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_output_tokens = settings.max_output_tokens,\n )\n}\n\n// Bedrock signs requests with AWS credentials, so `api_key_env` has no effect.\nfunction build_bedrock(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n aws.BedrockClient.new(\n model = spec.model,\n endpoint_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// One row for each provider. A new provider is a new row. The keys are the BAML provider prefixes.\nfunction client_builders() -> map<string, (ModelSpec, CallDefaults) -> ai.Client throws never> {\n {\n \"openai\": build_openai,\n \"openai-generic\": build_openai_generic,\n \"azure\": build_azure,\n \"openrouter\": build_openrouter,\n \"ollama\": build_ollama,\n \"anthropic\": build_anthropic,\n \"google\": build_google,\n \"vertex\": build_vertex,\n \"bedrock\": build_bedrock,\n }\n}\n\n// The provider prefixes that `ModelSpec.provider` accepts, in table order.\nfunction supported_providers() -> string[] {\n client_builders().keys()\n}\n\n// Builds the client of `spec` with the retry policy of the default clients.\nfunction build_client(spec: ModelSpec, defaults: CallDefaults) -> ai.Client {\n let builders = client_builders();\n let build = builders.get(spec.provider);\n if (build == null) {\n throw baml.errors.InvalidArgument {\n message: `unknown provider \"${spec.provider}\"; supported providers: ${supported_providers().join(\", \")}`,\n };\n };\n ai.clients.Retry.new(inner = build(spec, resolve_settings(spec, defaults)), max_attempts = 3)\n}\n\n// Builds and discards the client, so that Python can reject a bad provider before the first paid call.\nfunction check_model(spec: ModelSpec) -> bool {\n build_client(spec, short_defaults());\n true\n}\n\n// Precedence: `client` (the test hook) > `model` > null, which selects the client declared by the function.\nfunction select_client(\n client: ai.Client?,\n model: ModelSpec?,\n defaults: CallDefaults,\n) -> ai.Client? {\n if (client != null) {\n return client;\n };\n if (model == null) {\n return null;\n };\n build_client(model, defaults)\n}\n",
|
|
27
|
+
"model_spec_test.baml": "// Offline tests for model_spec.baml: building a client from a `ModelSpec` sends no request.\n\nfunction spec_for(provider: string) -> ModelSpec {\n ModelSpec {\n provider: provider,\n model: \"m\",\n api_key_env: null,\n base_url: null,\n temperature: null,\n max_output_tokens: null,\n headers: null,\n }\n}\n\ntest \"every provider of the table builds a client with no network access\" {\n let providers = client_builders().keys();\n assert.equal(providers.length(), 9);\n for (let provider in providers) {\n assert.equal(build_client(spec_for(provider), short_defaults()).requested_model(), \"m\");\n }\n}\n\ntest \"the table lists the BAML provider prefixes\" {\n let providers = supported_providers();\n for (\n let prefix in [\n \"openai\",\n \"openai-generic\",\n \"azure\",\n \"openrouter\",\n \"ollama\",\n \"anthropic\",\n \"google\",\n \"vertex\",\n \"bedrock\",\n ]\n ) {\n assert.is_true(providers.includes(prefix));\n }\n}\n\ntest \"a client with every optional field set builds\" {\n let spec = ModelSpec {\n provider: \"openai-generic\",\n model: \"m\",\n api_key_env: \"GATEWAY_KEY\",\n base_url: \"http://gateway.invalid/v1\",\n temperature: 0.2,\n max_output_tokens: 100,\n headers: { \"x-team\": \"ebm\" },\n };\n assert.equal(build_client(spec, short_defaults()).requested_model(), \"m\")\n}\n\ntest \"resolve_settings lets the user values win over the defaults of the call\" {\n let spec = ModelSpec {\n provider: \"openai\",\n model: \"m\",\n api_key_env: null,\n base_url: null,\n temperature: 0.1,\n max_output_tokens: 50,\n headers: null,\n };\n let settings = resolve_settings(spec, long_defaults());\n assert.equal(settings.temperature, 0.1);\n assert.equal(settings.max_output_tokens, 50)\n}\n\ntest \"resolve_settings takes the defaults of the call for null fields\" {\n let settings = resolve_settings(spec_for(\"openai\"), benchmark_defaults());\n assert.equal(settings.temperature, 0.0);\n assert.equal(settings.max_output_tokens, 8000)\n}\n\ntest \"the call defaults differ by call as the declared clients do\" {\n assert.equal(short_defaults().max_output_tokens, 6000);\n assert.equal(long_defaults().max_output_tokens, 8000);\n assert.equal(short_defaults().temperature, 0.7);\n assert.equal(benchmark_defaults().temperature, 0.0)\n}\n\n// The message of the error that `build_client` throws, or \"built\" if it throws none.\nfunction failure_message(provider: string) -> string {\n let built = build_client(spec_for(provider), short_defaults()) catch (e) {\n baml.errors.InvalidArgument => {\n return e.message;\n },\n };\n \"built\"\n}\n\n// The provider and the model that a client asks for, or \"null\".\nfunction client_id(client: ai.Client?) -> string {\n if (client == null) {\n return \"null\";\n };\n `${client.provider()}/${client.requested_model()}`\n}\n\ntest \"an unknown provider throws InvalidArgument that lists the supported providers\" {\n let message = failure_message(\"nope\");\n assert.is_true(message.includes(\"nope\"));\n assert.is_true(message.includes(\"anthropic\"));\n assert.is_true(message.includes(\"openai-generic\"))\n}\n\ntest \"a known provider throws no error\" {\n assert.equal(failure_message(\"ollama\"), \"built\")\n}\n\ntest \"check_model accepts a known provider and rejects an unknown one\" {\n assert.is_true(check_model(spec_for(\"ollama\")));\n let outcome = check_model(spec_for(\"nope\")) catch (e) {\n baml.errors.InvalidArgument => false,\n };\n assert.equal(outcome, false)\n}\n\ntest \"select_client prefers the client over the model\" {\n let scripted = ScriptedClient { calls: 0 };\n let selected = select_client(scripted, spec_for(\"ollama\"), short_defaults());\n assert.equal(client_id(selected), \"scripted/scripted\")\n}\n\ntest \"select_client builds the client of the model\" {\n let selected = select_client(null, spec_for(\"ollama\"), short_defaults());\n assert.equal(client_id(selected), \"ollama/m\")\n}\n\ntest \"select_client returns null with no client and no model, so the declared client runs\" {\n assert.equal(client_id(select_client(null, null, short_defaults())), \"null\")\n}\n\ntest \"describe_graph_cot sends every turn to the client when both client and model are set\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_graph_cot(\"G\", 7, default_graph_options(), client = scripted, model = spec_for(\"ollama\"));\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 3)\n}\n\ntest \"an Azure client without base_url names the deployment after the model\" {\n let azure = build_azure(spec_for(\"azure\"), short_defaults());\n assert.equal(azure.deployment_id, \"m\")\n}\n",
|
|
28
|
+
"prompt_tests.baml": "// Live prompt tests: real requests to the default Router clients, so they need a real OPENROUTER_API_KEY.\n// They check properties that a good answer must have (no model numbers for a layperson, a sentence limit, the\n// requested answer format), not exact wording, because the free router is not deterministic. A failure on one run can\n// be noise: rerun it before you change a prompt. Offline runs exclude them: `baml test -x root::Live`.\n\n// Number of sentences in `text`, counted by their closing punctuation.\nfunction count_sentences(text: string) -> int {\n let count = 0;\n for (let ch in text.split(\"\")) {\n if (ch == \".\" || ch == \"!\" || ch == \"?\") {\n count = count + 1;\n }\n }\n ;\n count\n}\n\nfunction live_local_facts() -> LocalFactTexts {\n LocalFactTexts {\n prediction: \"The model predicts 'loan denied' with probability 0.83 (log-odds +1.37).\",\n counterfactual: \"Smallest change that flips the prediction: income from 21000 to 48000 (score change -2.91 log-odds). The flip holds at the unfavourable edge of the confidence interval.\",\n sufficient_reason: \"Sufficient reason: income = 21000 and debt_ratio = 0.62 fix the prediction. Age (score +0.04) cannot change it.\",\n completions: \"Share of people with the same prediction: knowing only income 21000: 71%; income 21000 and debt_ratio 0.62: 100%.\",\n }\n}\n\ntestset \"Live\" {\n test \"layperson counterfactual: no model numbers, within the sentence limit, names the input\" {\n let text = LocalCounterfactual(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_local_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n assert.is_true(text.includes(\"income\"));\n assert.is_true(!text.includes(\"0.83\") && !text.includes(\"1.37\") && !text.includes(\"2.91\"));\n assert.is_true(!text.includes(\"%\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"layperson sufficient reason: says the other inputs cannot change the outcome\" {\n let text = LocalSufficientReason(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_local_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n assert.is_true(text.to_lower_case().includes(\"income\"));\n assert.is_true(text.to_lower_case().includes(\"age\"));\n assert.is_true(!text.includes(\"0.04\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"layperson completions: approximate wording and no percentages\" {\n let text = LocalCompletions(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_local_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n assert.is_true(!text.includes(\"71\") && !text.includes(\"%\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"data scientist counterfactual: keeps the figures from the facts\" {\n let text = LocalCounterfactual(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_local_facts(),\n \"data_scientist\",\n 3,\n default_graph_options(),\n );\n assert.is_true(text.includes(\"48000\") || text.includes(\"2.91\"))\n }\n\n test \"explain_local: three explanations, each non-empty, usage reported\" {\n let result = explain_local(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_local_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n assert.is_true(result.counterfactual.length() > 0);\n assert.is_true(result.sufficient_reason.length() > 0);\n assert.is_true(result.completions.length() > 0);\n assert.is_true(result.usage.input_tokens > 0);\n assert.is_true(result.usage.output_tokens > 0)\n }\n\n test \"describe_graph_cot: names the feature and respects the sentence limit\" {\n let graph = \"{\\\"feature_name\\\": \\\"age\\\", \\\"type\\\": \\\"continuous\\\", \\\"mean\\\": {\\\"18-30\\\": 0.8, \\\"30-50\\\": 0.0, \\\"50-90\\\": -0.6}}\";\n let result = describe_graph_cot(graph, 3, default_graph_options());\n assert.is_true(result.text.to_lower_case().includes(\"age\"));\n assert.is_true(count_sentences(result.text) <= 4);\n assert.is_true(result.usage.input_tokens > 0)\n }\n\n test \"describe_graph_cot: a multiclass graph is read class against class\" {\n // \"young\" leads at low ages and \"old\" at high ages; a good answer names both classes and the change of lead\n let graph = \"Feature Name: age\\nFeature Type: continuous\\nClass: young\\nMeans: {\\\"(18, 30)\\\": 1.5, \\\"(30, 50)\\\": 0.0, \\\"(50, 90)\\\": -1.5}\\nClass: middle\\nMeans: {\\\"(18, 30)\\\": 0.0, \\\"(30, 50)\\\": 0.5, \\\"(50, 90)\\\": 0.0}\\nClass: old\\nMeans: {\\\"(18, 30)\\\": -1.5, \\\"(30, 50)\\\": -0.5, \\\"(50, 90)\\\": 1.5}\";\n let result = describe_graph_cot(graph, 4, default_graph_options());\n let text = result.text.to_lower_case();\n assert.is_true(text.includes(\"young\") && text.includes(\"old\"));\n assert.is_true(count_sentences(result.text) <= 5)\n }\n\n test \"describe_graph_cot: a ModelSpec sends the requests to the chosen model\" {\n let graph = \"{\\\"feature_name\\\": \\\"age\\\", \\\"type\\\": \\\"continuous\\\", \\\"mean\\\": {\\\"18-30\\\": 0.8, \\\"30-50\\\": 0.0, \\\"50-90\\\": -0.6}}\";\n let model = ModelSpec {\n provider: \"openrouter\",\n model: \"openrouter/free\",\n api_key_env: \"OPENROUTER_API_KEY\",\n base_url: null,\n temperature: null,\n max_output_tokens: null,\n headers: null,\n };\n let result = describe_graph_cot(graph, 3, default_graph_options(), model = model);\n assert.is_true(result.text.to_lower_case().includes(\"age\"));\n assert.is_true(result.usage.output_tokens > 0)\n }\n\n test \"answer_benchmark_question: the final turn follows the requested format\" {\n let question = \"A model's graph for feature x has mean value 2.0 for x < 5 and -1.0 for x >= 5. Is the function decreasing in x? Think step by step.\";\n let result = answer_benchmark_question(question, \"'yes' or 'no'\");\n let answer = result.text.trim().to_lower_case();\n assert.is_true(answer.includes(\"yes\"));\n assert.is_true(answer.length() < 20)\n }\n}\n",
|
|
29
|
+
"talk_to_ebm.baml": "// Port of TalkToEBM's t2ebm/prompts.py and t2ebm/llm.py: prompts, chat clients, CoT pipelines, token accounting.\n\n// Some models behind `openrouter/free` reason mandatorily (reasoning cannot be disabled), and the hidden reasoning tokens\n// count against `max_tokens`, so the caps are generous.\n// Transient failures (rate limits, network, 5xx) are retried; rejected requests are not.\nclient RouterBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 6000);\n\n// First assistant turn of each conversation allows a longer answer.\nclient RouterLongBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 8000);\n\nclient Router = ai.clients.Retry.new(inner = RouterBase, max_attempts = 3);\n\n// The benchmark is scored on one answer per question, so it samples greedily, as TalkToEBM's benchmark does.\nclient RouterBenchmarkBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.0, max_tokens = 8000);\n\nclient RouterBenchmark = ai.clients.Retry.new(inner = RouterBenchmarkBase, max_attempts = 3);\n\nclient RouterLong = ai.clients.Retry.new(inner = RouterLongBase, max_attempts = 3);\n\nclass GraphPromptOptions {\n expert_description: string,\n graph_description: string,\n dataset_description: string,\n task_description: string,\n}\n\nfunction default_graph_options() -> GraphPromptOptions {\n GraphPromptOptions {\n expert_description: \"an expert statistician and data scientist\",\n graph_description: \"\",\n dataset_description: \"\",\n task_description: \"Describe the general pattern of the graph.\",\n }\n}\n\n// Prompt text helpers. Pure functions, so the Jinja `{% if %}` blocks become plain BAML.\n\nfunction graph_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You interpret global explanations produced by a Generalized Additive Model (GAM). Base every answer on the graph from the user, on any other information you are given, and on your knowledge of the real world. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// How to read the series of the classes. Only the graph of a multiclass classifier has \"Class: \" lines, so the prompt\n// of a binary classifier or a regressor gets no text about classes.\nfunction multiclass_format(graph: string) -> string {\n if (graph.includes(\"Class: \")) {\n return \"\\nA graph of a multiclass classifier repeats items 3 to 5 once for each class, after a line \\\"Class: <name>\\\". In every bin, the scores have the same mean over all the classes of the model, including classes that the graph leaves out. Only the differences between the classes change the predicted probabilities, so compare the classes with each other within each bin.\";\n };\n \"\"\n}\n\nfunction describe_graph_msg(graph: string, opts: GraphPromptOptions) -> string {\n let msg = `\n <graph_format>\n The graph is a JSON object. Its keys are the x-axis and its values are the y-axis. For a continuous feature, each key is an interval where the function predicts the same value. For a categorical feature, each key is a possible value of the feature.\n The graph contains these items:\n 1. The name of the feature.\n 2. The type of the feature (continuous, categorical, or boolean).\n 3. The mean values.\n 4. The lower bounds of the confidence interval (optional).\n 5. The upper bounds of the confidence interval (optional).${multiclass_format(graph)}\n </graph_format>\n\n <graph>\n ${graph}\n </graph>\n `\n + \"\\n\\n\";\n if (opts.graph_description.length() > 0) {\n msg = msg + `<graph_description>\\n${opts.graph_description}\\n</graph_description>\\n\\n`;\n };\n if (opts.dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${opts.dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg + `<task>\\n${opts.task_description}\\n</task>`\n}\n\nfunction summarize_ebm_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You provide an overall summary of a Generalized Additive Model (GAM). The model consists of different graphs, and each graph shows the effect of one input feature. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\nfunction summarize_ebm_msg(\n feature_importances: string,\n graph_descriptions: string,\n dataset_description: string,\n) -> string {\n let msg = `\n <feature_importances>\n ${feature_importances}\n </feature_importances>\n\n <graph_descriptions>\n ${graph_descriptions}\n </graph_descriptions>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + `\n <instructions>\n Summarize the model. The feature importances are global, and there is exactly one graph description for each feature.\n Cover the most important features and their effect on the outcome. Leave out unimportant features and effects, so the summary stays short.\n Include any surprising pattern, because the reader wants to know where the model differs from expectation.\n </instructions>\n `\n}\n\n// Request text shared by the second turn and the replay of that turn in the third.\nfunction surprises_request() -> string {\n `\n <instructions>\n Study the graph carefully. Highlight any region that you find surprising or counterintuitive, and name its range or category so the reader can find it. Suggest why the behavior is surprising and what may have caused it.\n </instructions>\n `\n}\n\nfunction final_description_request(num_sentences: int) -> string {\n `\n <instructions>\n Describe the graph in at most ${num_sentences} sentences.\n Include the important surprising patterns, because the reader uses them to decide where to look closer.\n The reader knows that the graph is from a Generalized Additive Model (GAM), so leave that background out.\n </instructions>\n `\n}\n\n// describe_graph_cot, turn 1: system + describe_graph prompt.\nfunction GraphFirstPass(graph: string, opts: GraphPromptOptions) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n `\n}\n\n// Turn 2: surprising regions. `first_pass` is the assistant reply from turn 1.\nfunction GraphSurprises(graph: string, first_pass: string, opts: GraphPromptOptions) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n `\n}\n\n// Turn 3: brief final description.\nfunction GraphFinalDescription(\n graph: string,\n first_pass: string,\n surprises: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n ${role(\"assistant\")}\n ${surprises}\n ${role(\"user\")}\n ${final_description_request(num_sentences)}\n `\n}\n\n// summarize_ebm, turn 1.\nfunction EbmSummaryDraft(\n feature_importances: string,\n graph_descriptions: string,\n opts: GraphPromptOptions,\n) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description)}\n `\n}\n\n// summarize_ebm, turn 2: shorten the draft.\nfunction EbmSummaryShorten(\n feature_importances: string,\n graph_descriptions: string,\n draft: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description)}\n ${role(\"assistant\")}\n ${draft}\n ${role(\"user\")}\n <instructions>\n Shorten your summary to at most ${num_sentences} sentences. Keep the most important information, because the reader wants the main effects and surprises.\n </instructions>\n `\n}\n\n// Token accounting.\n\nclass TokenUsage {\n input_tokens: int,\n output_tokens: int,\n}\n\n// An LLM result plus the tokens the provider reported for producing it.\nclass Counted {\n text: string,\n usage: TokenUsage,\n}\n\nclass TokenBudget {\n estimated_tokens: int,\n status: \"ok\" | \"warn\" | \"over\",\n}\n\nfunction add_usage(a: TokenUsage, b: TokenUsage) -> TokenUsage {\n TokenUsage {\n input_tokens: a.input_tokens + b.input_tokens,\n output_tokens: a.output_tokens + b.output_tokens,\n }\n}\n\n// Rough estimate (~3 chars/token): graph text is mostly digits and punctuation, which tokenize worse than prose.\n// No tokenizer is available before a call, so this is a guard rail; the provider's own error is the real limit.\nfunction estimate_tokens(text: string) -> int {\n (text.length() + 2) / 3\n}\n\n// \"ok\" within `max_tokens`; \"warn\" up to `max_tokens` plus `overbudget_percent`; \"over\" beyond that.\nfunction check_token_budget(text: string, max_tokens: int, overbudget_percent: int) -> TokenBudget {\n let estimated = estimate_tokens(text);\n let hard_limit = max_tokens * (100 + overbudget_percent) / 100;\n if (estimated > hard_limit) {\n return TokenBudget { estimated_tokens: estimated, status: \"over\" };\n };\n if (estimated > max_tokens) {\n return TokenBudget { estimated_tokens: estimated, status: \"warn\" };\n };\n TokenBudget { estimated_tokens: estimated, status: \"ok\" }\n}\n\n// Runs one text-producing LLM function and keeps the provider's token usage. A count that the provider did not report\n// is zero.\nfunction run_counted(spec: ai.FunctionSpec<string>, client: ai.Client? = null) -> Counted {\n let result = ai.Agent.new(client = client).run(spec);\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0 };\n for (let entry in result.usage) {\n usage\n = add_usage(\n usage,\n TokenUsage {\n input_tokens: entry.usage.input_tokens ?? 0,\n output_tokens: entry.usage.output_tokens ?? 0,\n },\n );\n }\n Counted { text: result.value, usage: usage }\n}\n\n// Entry points called from Python.\n\nfunction describe_graph_cot(\n graph: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n //# Three sequential assistant turns, each seeing the previous replies\n let long = select_client(client, model, long_defaults());\n let short = select_client(client, model, short_defaults());\n let first_pass = run_counted(GraphFirstPass@spec(graph, opts), client = long);\n let surprises = run_counted(GraphSurprises@spec(graph, first_pass.text, opts), client = short);\n let final = run_counted(\n GraphFinalDescription@spec(graph, first_pass.text, surprises.text, num_sentences, opts),\n client = short,\n );\n Counted {\n text: final.text,\n usage: add_usage(add_usage(first_pass.usage, surprises.usage), final.usage),\n }\n}\n\n// `num_sentences` null keeps the draft summary as is.\nfunction summarize_ebm(\n feature_importances: string,\n graph_descriptions: string,\n num_sentences: int?,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let draft = run_counted(\n EbmSummaryDraft@spec(feature_importances, graph_descriptions, opts),\n client = select_client(client, model, long_defaults()),\n );\n if (num_sentences == null) {\n return draft;\n };\n let shortened = run_counted(\n EbmSummaryShorten@spec(feature_importances, graph_descriptions, draft.text, num_sentences, opts),\n client = select_client(client, model, short_defaults()),\n );\n Counted { text: shortened.text, usage: add_usage(draft.usage, shortened.usage) }\n}\n\n// Full EBM description: one chain per feature graph (concurrently), then the summary.\n// `graphs[i]` is the text of the graph of `feature_names[i]`.\nfunction describe_ebm(\n feature_names: string[],\n graphs: string[],\n feature_importances: string,\n num_sentences: int,\n graph_sentences: int,\n max_concurrency: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n if (feature_names.length() != graphs.length()) {\n throw baml.errors.InvalidArgument { message: \"feature_names and graphs must have the same length\" };\n };\n //# One description chain per graph, at most `max_concurrency` at a time\n let limit = baml.spawn.Limit.new(max_concurrency);\n let futures = graphs.map((graph) -> {\n spawn with limit {\n describe_graph_cot(graph, graph_sentences, opts, client = client, model = model)\n }\n });\n let described: Counted[] = await baml.future.all(futures);\n\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0 };\n let lines: string[] = [];\n let i = 0;\n while (i < described.length()) {\n let item = described[i];\n usage = add_usage(usage, item.usage);\n lines.push(`${feature_names[i]}: ${item.text}`);\n i = i + 1;\n }\n //# Summarize all the per-graph descriptions\n let summary = summarize_ebm(feature_importances, lines.join(\"\\n\\n\"), num_sentences, opts, client = client, model = model);\n Counted { text: summary.text, usage: add_usage(usage, summary.usage) }\n}\n\n// Benchmark (benchmark/): the questions end with \"Think step by step.\", so turn 1 reasons freely and turn 2\n// extracts the final answer in the format the scorer parses.\n\nfunction BenchmarkReasoning(question: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n `\n}\n\nfunction BenchmarkAnswer(question: string, reasoning: string, answer_format: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n ${role(\"assistant\")}\n ${reasoning}\n ${role(\"user\")}\n <instructions>\n Summarize your response by answering with ${answer_format}. Reply with the answer only, because a program parses your reply.\n </instructions>\n `\n}\n\nfunction answer_benchmark_question(\n question: string,\n answer_format: string,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let selected = select_client(client, model, benchmark_defaults());\n let reasoning = run_counted(BenchmarkReasoning@spec(question), client = selected);\n let answer = run_counted(BenchmarkAnswer@spec(question, reasoning.text, answer_format), client = selected);\n Counted { text: answer.text, usage: add_usage(reasoning.usage, answer.usage) }\n}\n",
|
|
30
|
+
"talk_to_ebm_test.baml": "// Offline tests for talk_to_ebm.baml: pure helpers and the pipelines on a scripted client.\n\n// A model turn that holds `text` and reports `input_tokens` and `output_tokens`. The runner reads the usage of a turn\n// from its calls.\nfunction scripted_turn(text: string, input_tokens: int, output_tokens: int) -> ai.ModelTurn {\n ai.ModelTurn {\n parsed_output: null,\n content: [ai.content.Text { text: text }],\n stop_reason: ai.content.StopReason.Complete,\n resolved_model: null,\n requested_model: null,\n calls: [\n ai.events.ModelCall {\n resolved_model: ai.events.ModelKey { provider: \"scripted\", model: null },\n usage: ai.events.Usage {\n input_tokens: input_tokens,\n output_tokens: output_tokens,\n cached_input_tokens: null,\n reasoning_tokens: null,\n cache_write_input_tokens: null,\n uncached_input_tokens: null,\n },\n timing: ai.events.Timing { start_time: null, duration: null, ttft: null },\n http_request: null,\n http_response: null,\n sse_responses: null,\n },\n ],\n child_turns: [],\n error: null,\n }\n}\n\n// Answers every call with \"R\" and reports 10 input / 5 output tokens; counts the calls.\nclass ScriptedClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"scripted\"\n }\n\n function requested_model(self) -> string {\n \"scripted\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://scripted.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n scripted_turn(\"R\", 10, 5)\n }\n }\n}\n\ntest \"estimate_tokens rounds up at 3 chars per token\" {\n assert.equal(estimate_tokens(\"\"), 0);\n assert.equal(estimate_tokens(\"abc\"), 1);\n assert.equal(estimate_tokens(\"abcd\"), 2)\n}\n\ntest \"check_token_budget: ok, warn, over\" {\n // 300 chars -> 100 tokens estimated\n let text = \"x\".repeat(300);\n assert.equal(check_token_budget(text, 100, 50).status, \"ok\");\n assert.equal(check_token_budget(text, 80, 50).status, \"warn\");\n assert.equal(check_token_budget(text, 60, 50).status, \"over\");\n assert.equal(check_token_budget(text, 80, 50).estimated_tokens, 100)\n}\n\ntest \"add_usage sums both counters\" {\n let total = add_usage(\n TokenUsage { input_tokens: 1, output_tokens: 2 },\n TokenUsage { input_tokens: 10, output_tokens: 20 },\n );\n assert.equal(total.input_tokens, 11);\n assert.equal(total.output_tokens, 22)\n}\n\ntest \"describe_graph_msg includes optional sections only when set\" {\n let bare = describe_graph_msg(\"GRAPH\", default_graph_options());\n assert.is_true(bare.includes(\"GRAPH\"));\n assert.is_true(bare.includes(\"Describe the general pattern of the graph.\"));\n assert.is_true(!bare.includes(\"<dataset_description>\"));\n\n let opts = GraphPromptOptions { ...default_graph_options(), dataset_description: \"A pneumonia dataset\" };\n assert.is_true(describe_graph_msg(\"GRAPH\", opts).includes(\"A pneumonia dataset\"))\n}\n\ntest \"describe_graph_msg says how to read the classes of a multiclass graph\" {\n let msg = describe_graph_msg(\"Feature Name: a\\nClass: high\\nMeans: {}\", default_graph_options());\n assert.is_true(\n msg.includes(\"(optional).\\nA graph of a multiclass classifier repeats items 3 to 5\"),\n );\n assert.is_true(msg.includes(\"including classes that the graph leaves out\"));\n assert.is_true(msg.includes(\"compare the classes with each other within each bin.\"));\n assert.is_true(position_of(msg, \"Class: <name>\") < position_of(msg, \"</graph_format>\"))\n}\n\ntest \"describe_graph_msg of a graph with one output says nothing about classes\" {\n let msg = describe_graph_msg(\"Feature Name: a\\nMeans: {}\", default_graph_options());\n assert.is_true(!msg.includes(\"multiclass\"));\n assert.is_true(\n msg.includes(\"5. The upper bounds of the confidence interval (optional).\\n</graph_format>\"),\n )\n}\n\ntest \"describe_graph_cot makes three calls and sums usage\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = scripted);\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 3);\n assert.equal(result.usage.input_tokens, 30);\n assert.equal(result.usage.output_tokens, 15)\n}\n\ntest \"summarize_ebm shortens only when num_sentences is set\" {\n let draft_only = ScriptedClient { calls: 0 };\n summarize_ebm(\"IMP\", \"DESC\", null, default_graph_options(), client = draft_only);\n assert.equal(draft_only.calls, 1);\n\n let shortened = ScriptedClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = shortened);\n assert.equal(shortened.calls, 2);\n assert.equal(result.usage.input_tokens, 20)\n}\n\ntest \"describe_ebm: one chain per graph plus the summary\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_ebm([\"age\", \"bmi\"], [\"G1\", \"G2\"], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted);\n // 2 graphs x 3 turns + draft + shorten. The two chains run at the same time, so `self.calls += 1` of the\n // scripted client can lose an increment; each call reports 10 input and 5 output tokens, so the usage counts the\n // calls without a shared counter.\n assert.equal(result.usage.input_tokens, 8 * 10);\n assert.equal(result.usage.output_tokens, 8 * 5)\n}\n\ntest \"describe_ebm rejects mismatched names and graphs\" {\n let scripted = ScriptedClient { calls: 0 };\n let outcome = describe_ebm([\"age\"], [], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted) catch (e) {\n baml.errors.InvalidArgument => null,\n };\n assert.is_true(outcome == null);\n assert.equal(scripted.calls, 0)\n}\n\ntest \"answer_benchmark_question reasons, then extracts the answer\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = scripted);\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 2);\n assert.equal(result.usage.input_tokens, 20);\n assert.equal(result.usage.output_tokens, 10)\n}\n\ntest \"local_user_msg prepends the global explanation and follows the audience\" {\n let layperson = local_user_msg(\"GLOBAL\", \"PRED\", \"QUESTION\", \"FACTS\", \"layperson\", \"\");\n assert.is_true(layperson.includes(\"GLOBAL\"));\n assert.is_true(layperson.includes(\"FACTS\"));\n assert.is_true(layperson.includes(\"model numbers\"));\n assert.is_true(!layperson.includes(\"<dataset_description>\"));\n\n let expert = local_user_msg(\"GLOBAL\", \"PRED\", \"QUESTION\", \"FACTS\", \"data_scientist\", \"A bank dataset\");\n assert.is_true(expert.includes(\"confidence intervals\"));\n assert.is_true(expert.includes(\"A bank dataset\"))\n}\n\ntest \"explain_local makes three calls and sums usage\" {\n let scripted = ScriptedClient { calls: 0 };\n let facts = LocalFactTexts {\n prediction: \"P\",\n counterfactual: \"C\",\n sufficient_reason: \"S\",\n completions: \"K\",\n };\n let result = explain_local(\"GLOBAL\", facts, \"layperson\", 5, default_graph_options(), client = scripted);\n assert.equal(scripted.calls, 3);\n assert.equal(result.counterfactual, \"R\");\n assert.equal(result.sufficient_reason, \"R\");\n assert.equal(result.completions, \"R\");\n assert.equal(result.usage.input_tokens, 30);\n assert.equal(result.usage.output_tokens, 15)\n}\n\n// Answers every call with its own rendered prompt, so a test can see what each call was given.\nclass EchoClient {\n implements ai.Client {\n function provider(self) -> string {\n \"echo\"\n }\n\n function requested_model(self) -> string {\n \"echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n let prompt = input.prompt(ai.internal.build_output_format(input.output_type));\n scripted_turn(prompt.text(), 1, 1)\n }\n }\n}\n\n// Index of `needle` in `text`, or -1 when absent, so positions can be ordered.\nfunction position_of(text: string, needle: string) -> int {\n let found = text.index_of(needle);\n if (found == null) {\n return -1;\n };\n found\n}\n\ntest \"local_system_msg names the expert and forbids invented facts\" {\n let msg = local_system_msg(\"a credit risk analyst\");\n assert.is_true(msg.includes(\"You are a credit risk analyst.\"));\n assert.is_true(msg.includes(\"invented number\"))\n}\n\ntest \"audience_guidance differs per audience\" {\n let layperson = audience_guidance(\"layperson\");\n let expert = audience_guidance(\"data_scientist\");\n assert.is_true(layperson.includes(\"not technical\"));\n assert.is_true(!layperson.includes(\"data scientist\"));\n assert.is_true(expert.includes(\"data scientist\"));\n assert.is_true(!expert.includes(\"not technical\"))\n}\n\ntest \"local_user_msg orders the global explanation, dataset, prediction, facts, guidance and question\" {\n let msg = local_user_msg(\"GLOBAL\", \"PRED\", \"QUESTION\", \"FACTS\", \"layperson\", \"DATASET\");\n let positions = [\n position_of(msg, \"GLOBAL\"),\n position_of(msg, \"DATASET\"),\n position_of(msg, \"PRED\"),\n position_of(msg, \"FACTS\"),\n position_of(msg, \"not technical\"),\n position_of(msg, \"QUESTION\"),\n ];\n assert.is_true(positions[0] >= 0);\n assert.is_true(positions[0] < positions[1]);\n assert.is_true(positions[1] < positions[2]);\n assert.is_true(positions[2] < positions[3]);\n assert.is_true(positions[3] < positions[4]);\n assert.is_true(positions[4] < positions[5])\n}\n\ntest \"question functions state the sentence limit and their own query\" {\n assert.is_true(counterfactual_question(3).includes(\"at most 3 sentences\"));\n assert.is_true(counterfactual_question(3).includes(\"minimum change required\"));\n assert.is_true(sufficient_reason_question(4).includes(\"at most 4 sentences\"));\n assert.is_true(sufficient_reason_question(4).includes(\"sufficient reason\"));\n assert.is_true(completions_question(6).includes(\"at most 6 sentences\"));\n assert.is_true(completions_question(6).includes(\"count completions\"))\n}\n\ntest \"each local function gets its own facts, question and system role\" {\n let facts = LocalFactTexts {\n prediction: \"PRED-FACT\",\n counterfactual: \"CF-FACT\",\n sufficient_reason: \"SR-FACT\",\n completions: \"CO-FACT\",\n };\n let opts = GraphPromptOptions { ...default_graph_options(), expert_description: \"an EXPERT\" };\n\n let cf = LocalCounterfactual@spec(\"GLOBAL\", facts, \"layperson\", 3, opts).prompt().text();\n let sr = LocalSufficientReason@spec(\"GLOBAL\", facts, \"layperson\", 4, opts).prompt().text();\n let co = LocalCompletions@spec(\"GLOBAL\", facts, \"layperson\", 6, opts).prompt().text();\n\n for (let prompt in [cf, sr, co]) {\n assert.is_true(prompt.includes(\"You are an EXPERT.\"));\n assert.is_true(prompt.includes(\"GLOBAL\"));\n assert.is_true(prompt.includes(\"PRED-FACT\"))\n }\n ;\n\n assert.is_true(\n cf.includes(\"CF-FACT\")\n && cf.includes(\"at most 3 sentences\")\n && cf.includes(\"minimum change required\"),\n );\n assert.is_true(!cf.includes(\"SR-FACT\") && !cf.includes(\"CO-FACT\"));\n assert.is_true(\n sr.includes(\"SR-FACT\")\n && sr.includes(\"at most 4 sentences\")\n && sr.includes(\"sufficient reason\"),\n );\n assert.is_true(!sr.includes(\"CF-FACT\") && !sr.includes(\"CO-FACT\"));\n assert.is_true(\n co.includes(\"CO-FACT\")\n && co.includes(\"at most 6 sentences\")\n && co.includes(\"count completions\"),\n );\n assert.is_true(!co.includes(\"CF-FACT\") && !co.includes(\"SR-FACT\"))\n}\n\ntest \"explain_local routes each answer to its own field\" {\n let facts = LocalFactTexts {\n prediction: \"PRED-FACT\",\n counterfactual: \"CF-FACT\",\n sufficient_reason: \"SR-FACT\",\n completions: \"CO-FACT\",\n };\n let result = explain_local(\"GLOBAL\", facts, \"data_scientist\", 5, default_graph_options(), client = EchoClient {});\n assert.is_true(result.counterfactual.includes(\"CF-FACT\"));\n assert.is_true(\n !result.counterfactual.includes(\"SR-FACT\") && !result.counterfactual.includes(\"CO-FACT\"),\n );\n assert.is_true(result.sufficient_reason.includes(\"SR-FACT\"));\n assert.is_true(\n !result.sufficient_reason.includes(\"CF-FACT\")\n && !result.sufficient_reason.includes(\"CO-FACT\"),\n );\n assert.is_true(result.completions.includes(\"CO-FACT\"));\n assert.is_true(\n !result.completions.includes(\"CF-FACT\") && !result.completions.includes(\"SR-FACT\"),\n );\n assert.is_true(result.completions.includes(\"data scientist\"));\n assert.equal(result.usage.input_tokens, 3);\n assert.equal(result.usage.output_tokens, 3)\n}\n\n// Answers call n with \"REPLY-n\" and counts the calls, so a test can see which reply reached which later prompt.\n// Use it only for sequential pipelines: concurrent calls would race on the counter.\nclass SequenceClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"sequence\"\n }\n\n function requested_model(self) -> string {\n \"sequence\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://sequence.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n scripted_turn(`REPLY-${self.calls}`, 1, 1)\n }\n }\n}\n\n// The roles of a rendered prompt, joined, e.g. \"system,user,assistant,user\".\nfunction roles_of(prompt: ai.Prompt) -> string {\n prompt.messages()\n .map((m) -> {\n m.role\n })\n .join(\",\")\n}\n\n// The content of message `index` of a rendered prompt.\nfunction message_at(prompt: ai.Prompt, index: int) -> string {\n prompt.messages()[index].content\n}\n\ntest \"default_graph_options is a plain statistician with no extra context\" {\n let opts = default_graph_options();\n assert.equal(opts.expert_description, \"an expert statistician and data scientist\");\n assert.equal(opts.graph_description, \"\");\n assert.equal(opts.dataset_description, \"\");\n assert.equal(opts.task_description, \"Describe the general pattern of the graph.\")\n}\n\ntest \"estimate_tokens rounds up at every remainder\" {\n assert.equal(estimate_tokens(\"a\"), 1);\n assert.equal(estimate_tokens(\"ab\"), 1);\n assert.equal(estimate_tokens(\"abc\"), 1);\n assert.equal(estimate_tokens(\"abcd\"), 2);\n assert.equal(estimate_tokens(\"abcdef\"), 2);\n assert.equal(estimate_tokens(\"abcdefg\"), 3)\n}\n\ntest \"check_token_budget changes status exactly past the limits\" {\n // 300 chars -> 100 tokens. The hard limit is max_tokens * (100 + percent) / 100, rounded down.\n let text = \"x\".repeat(300);\n assert.equal(check_token_budget(text, 100, 0).status, \"ok\");\n assert.equal(check_token_budget(text, 99, 2).status, \"warn\");\n assert.equal(check_token_budget(text, 99, 1).status, \"over\");\n assert.equal(check_token_budget(text, 99, 0).status, \"over\");\n assert.equal(check_token_budget(text, 67, 50).status, \"warn\");\n assert.equal(check_token_budget(text, 66, 50).status, \"over\");\n assert.equal(check_token_budget(text, 66, 50).estimated_tokens, 100)\n}\n\ntest \"add_usage keeps the operands independent\" {\n let zero = TokenUsage { input_tokens: 0, output_tokens: 0 };\n let usage = TokenUsage { input_tokens: 3, output_tokens: 4 };\n assert.equal(add_usage(zero, usage).input_tokens, 3);\n assert.equal(add_usage(zero, usage).output_tokens, 4);\n assert.equal(add_usage(usage, usage).input_tokens, 6);\n assert.equal(add_usage(usage, usage).output_tokens, 8)\n}\n\ntest \"graph_system_msg names the expert and the GAM\" {\n let msg = graph_system_msg(\"a medical statistician\");\n assert.is_true(msg.includes(\"You are a medical statistician.\"));\n assert.is_true(msg.includes(\"Generalized Additive Model\"))\n}\n\ntest \"describe_graph_msg orders graph, graph description, dataset and task\" {\n let opts = GraphPromptOptions {\n expert_description: \"E\",\n graph_description: \"GRAPHDESC\",\n dataset_description: \"DATASET\",\n task_description: \"TASK\",\n };\n let msg = describe_graph_msg(\"GRAPH\", opts);\n assert.is_true(position_of(msg, \"GRAPH\") >= 0);\n assert.is_true(position_of(msg, \"GRAPH\") < position_of(msg, \"GRAPHDESC\"));\n assert.is_true(position_of(msg, \"GRAPHDESC\") < position_of(msg, \"DATASET\"));\n assert.is_true(position_of(msg, \"DATASET\") < position_of(msg, \"TASK\"));\n assert.is_true(msg.includes(\"<dataset_description>\"))\n}\n\ntest \"describe_graph_msg leaves out each optional section on its own\" {\n let only_graph = GraphPromptOptions { ...default_graph_options(), graph_description: \"GRAPHDESC\" };\n let msg = describe_graph_msg(\"GRAPH\", only_graph);\n assert.is_true(msg.includes(\"GRAPHDESC\"));\n assert.is_true(!msg.includes(\"<dataset_description>\"));\n\n let only_dataset = GraphPromptOptions { ...default_graph_options(), dataset_description: \"DATASET\" };\n let other = describe_graph_msg(\"GRAPH\", only_dataset);\n assert.is_true(other.includes(\"DATASET\"));\n assert.is_true(!other.includes(\"GRAPHDESC\"));\n\n let custom = GraphPromptOptions { ...default_graph_options(), task_description: \"MY TASK\" };\n let task = describe_graph_msg(\"GRAPH\", custom);\n assert.is_true(task.includes(\"MY TASK\"));\n assert.is_true(!task.includes(\"Describe the general pattern\"))\n}\n\ntest \"summarize_ebm_system_msg names the expert\" {\n let msg = summarize_ebm_system_msg(\"a risk analyst\");\n assert.is_true(msg.includes(\"You are a risk analyst.\"));\n assert.is_true(msg.includes(\"overall summary\"))\n}\n\ntest \"summarize_ebm_msg orders importances, descriptions and dataset, and the dataset is optional\" {\n let msg = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"DATASET\");\n assert.is_true(position_of(msg, \"IMPORTANCES\") >= 0);\n assert.is_true(position_of(msg, \"IMPORTANCES\") < position_of(msg, \"DESCRIPTIONS\"));\n assert.is_true(position_of(msg, \"DESCRIPTIONS\") < position_of(msg, \"DATASET\"));\n assert.is_true(position_of(msg, \"DATASET\") < position_of(msg, \"<instructions>\"));\n\n let bare = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"\");\n assert.is_true(!bare.includes(\"<dataset_description>\"));\n assert.is_true(bare.includes(\"<instructions>\"))\n}\n\ntest \"describe_graph_cot prompts: roles, content and sentence limit per turn\" {\n let opts = GraphPromptOptions { ...default_graph_options(), expert_description: \"an EXPERT\" };\n\n let first = GraphFirstPass@spec(\"GRAPH\", opts).prompt();\n assert.equal(roles_of(first), \"system,user\");\n assert.is_true(message_at(first, 0).includes(\"You are an EXPERT.\"));\n assert.is_true(message_at(first, 1).includes(\"GRAPH\"));\n\n let second = GraphSurprises@spec(\"GRAPH\", \"FIRST\", opts).prompt();\n assert.equal(roles_of(second), \"system,user,assistant,user\");\n assert.is_true(message_at(second, 1).includes(\"GRAPH\"));\n assert.equal(message_at(second, 2).trim(), \"FIRST\");\n assert.is_true(message_at(second, 3).includes(\"surprising or counterintuitive\"));\n\n let third = GraphFinalDescription@spec(\"GRAPH\", \"FIRST\", \"SURPRISES\", 7, opts).prompt();\n assert.equal(roles_of(third), \"system,user,assistant,user,assistant,user\");\n assert.equal(message_at(third, 2).trim(), \"FIRST\");\n assert.equal(message_at(third, 4).trim(), \"SURPRISES\");\n assert.is_true(message_at(third, 5).includes(\"at most 7 sentence\"))\n}\n\ntest \"describe_graph_cot feeds each reply to the next turn in order\" {\n let sequence = SequenceClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = sequence);\n assert.equal(sequence.calls, 3);\n assert.equal(result.text, \"REPLY-3\");\n assert.equal(result.usage.input_tokens, 3);\n assert.equal(result.usage.output_tokens, 3);\n\n // The echo client returns the last prompt, which must hold both earlier replies in order.\n let echo = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = EchoClient {});\n assert.is_true(echo.text.includes(\"GRAPH\"));\n assert.is_true(echo.text.includes(\"at most 7 sentence\"));\n assert.is_true(position_of(echo.text, \"Highlight any region\") >= 0);\n assert.is_true(\n position_of(echo.text, \"Describe the graph in at most\")\n > position_of(echo.text, \"Highlight any region\"),\n )\n}\n\ntest \"describe_graph_cot passes the second and third turns the replies of the earlier turns\" {\n let sequence = SequenceClient { calls: 0 };\n let first = GraphFirstPass@spec(\"GRAPH\", default_graph_options());\n let reply_one = run_counted(first, client = sequence);\n assert.equal(reply_one.text, \"REPLY-1\");\n let reply_two = run_counted(GraphSurprises@spec(\"GRAPH\", reply_one.text, default_graph_options()), client = sequence);\n assert.equal(reply_two.text, \"REPLY-2\");\n let third = GraphFinalDescription@spec(\"GRAPH\", reply_one.text, reply_two.text, 5, default_graph_options())\n .prompt();\n assert.equal(message_at(third, 2).trim(), \"REPLY-1\");\n assert.equal(message_at(third, 4).trim(), \"REPLY-2\")\n}\n\ntest \"summarize_ebm prompts: roles, content and sentence limit\" {\n let opts = GraphPromptOptions {\n ...default_graph_options(),\n expert_description: \"an EXPERT\",\n dataset_description: \"DATASET\",\n };\n let draft = EbmSummaryDraft@spec(\"IMPORTANCES\", \"DESCRIPTIONS\", opts).prompt();\n assert.equal(roles_of(draft), \"system,user\");\n assert.is_true(message_at(draft, 0).includes(\"You are an EXPERT.\"));\n assert.is_true(message_at(draft, 1).includes(\"IMPORTANCES\"));\n assert.is_true(message_at(draft, 1).includes(\"DESCRIPTIONS\"));\n assert.is_true(message_at(draft, 1).includes(\"DATASET\"));\n\n let shorten = EbmSummaryShorten@spec(\"IMPORTANCES\", \"DESCRIPTIONS\", \"DRAFT\", 12, opts).prompt();\n assert.equal(roles_of(shorten), \"system,user,assistant,user\");\n assert.is_true(message_at(shorten, 1).includes(\"IMPORTANCES\"));\n assert.equal(message_at(shorten, 2).trim(), \"DRAFT\");\n assert.is_true(message_at(shorten, 3).includes(\"at most 12 sentences\"))\n}\n\ntest \"summarize_ebm returns the draft when num_sentences is null, else the shortened reply\" {\n let draft_only = SequenceClient { calls: 0 };\n let kept = summarize_ebm(\"IMP\", \"DESC\", null, default_graph_options(), client = draft_only);\n assert.equal(kept.text, \"REPLY-1\");\n assert.equal(kept.usage.input_tokens, 1);\n assert.equal(kept.usage.output_tokens, 1);\n\n let shortened = SequenceClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = shortened);\n assert.equal(result.text, \"REPLY-2\");\n assert.equal(result.usage.input_tokens, 2);\n assert.equal(result.usage.output_tokens, 2);\n\n // The shorten prompt holds the draft and the limit.\n let echo = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = EchoClient {});\n assert.is_true(echo.text.includes(\"at most 30 sentences\"));\n assert.is_true(echo.text.includes(\"IMP\"));\n assert.is_true(echo.text.includes(\"DESC\"))\n}\n\ntest \"describe_ebm pairs each feature name with its own description, in order\" {\n let result = describe_ebm(\n [\"age\", \"bmi\"],\n [\"G-AGE\", \"G-BMI\"],\n \"IMP\",\n 30,\n 7,\n 2,\n default_graph_options(),\n client = EchoClient {},\n );\n // The summary draft echoes the \"name: description\" lines; each description echoes its own graph.\n assert.is_true(position_of(result.text, \"age: \") >= 0);\n assert.is_true(position_of(result.text, \"age: \") < position_of(result.text, \"bmi: \"));\n assert.is_true(position_of(result.text, \"G-AGE\") < position_of(result.text, \"bmi: \"));\n assert.is_true(position_of(result.text, \"bmi: \") < position_of(result.text, \"G-BMI\"));\n assert.is_true(result.text.includes(\"at most 30 sentences\"));\n assert.is_true(result.text.includes(\"at most 7 sentence\"));\n assert.is_true(result.text.includes(\"IMP\"))\n}\n\ntest \"describe_ebm gives the same usage whatever the concurrency\" {\n let one = describe_ebm(\n [\"a\", \"b\", \"c\"],\n [\"G1\", \"G2\", \"G3\"],\n \"IMP\",\n 30,\n 7,\n 1,\n default_graph_options(),\n client = EchoClient {},\n );\n let many = describe_ebm(\n [\"a\", \"b\", \"c\"],\n [\"G1\", \"G2\", \"G3\"],\n \"IMP\",\n 30,\n 7,\n 8,\n default_graph_options(),\n client = EchoClient {},\n );\n // 3 graphs x 3 turns + draft + shorten = 11 calls at 1 input and 1 output token each.\n assert.equal(one.usage.input_tokens, 11);\n assert.equal(one.usage.output_tokens, 11);\n assert.equal(many.usage.input_tokens, 11);\n assert.equal(many.usage.output_tokens, 11)\n}\n\ntest \"describe_ebm with no graphs only summarizes\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_ebm([], [], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted);\n assert.equal(scripted.calls, 2);\n assert.equal(result.usage.input_tokens, 20)\n}\n\ntest \"describe_ebm rejects more names than graphs and more graphs than names\" {\n let scripted = ScriptedClient { calls: 0 };\n let more_graphs = describe_ebm([], [\"G\"], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted) catch (e) {\n baml.errors.InvalidArgument => null,\n };\n assert.is_true(more_graphs == null);\n assert.equal(scripted.calls, 0)\n}\n\ntest \"benchmark prompts: roles, question, reasoning and answer format\" {\n let reasoning = BenchmarkReasoning@spec(\"QUESTION\").prompt();\n assert.equal(roles_of(reasoning), \"system,user\");\n assert.is_true(\n message_at(reasoning, 0).includes(\"You are an expert statistician and data scientist.\"),\n );\n assert.is_true(message_at(reasoning, 1).includes(\"QUESTION\"));\n\n let answer = BenchmarkAnswer@spec(\"QUESTION\", \"REASONING\", \"'a' or 'b'\").prompt();\n assert.equal(roles_of(answer), \"system,user,assistant,user\");\n assert.is_true(message_at(answer, 1).includes(\"QUESTION\"));\n assert.equal(message_at(answer, 2).trim(), \"REASONING\");\n assert.is_true(message_at(answer, 3).includes(\"'a' or 'b'\"));\n assert.is_true(message_at(answer, 3).includes(\"Reply with the answer only\"))\n}\n\ntest \"answer_benchmark_question hands the reasoning to the answer turn\" {\n let sequence = SequenceClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = sequence);\n assert.equal(sequence.calls, 2);\n assert.equal(result.text, \"REPLY-2\");\n assert.equal(result.usage.input_tokens, 2);\n assert.equal(result.usage.output_tokens, 2);\n\n let echo = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = EchoClient {});\n assert.is_true(echo.text.includes(\"'a' or 'b'\"));\n assert.is_true(echo.text.includes(\"QUESTION\"));\n assert.is_true(echo.text.includes(\"Summarize your response\"))\n}\n\n// A model turn that holds `text` and reports 1 input and 1 output token.\nfunction text_turn(text: string) -> ai.ModelTurn {\n scripted_turn(text, 1, 1)\n}\n\n// Answers call n with \"Rn<<\" + its rendered prompt + \">>\". A later prompt then shows which earlier reply sits where.\n// Use it only for sequential pipelines: concurrent calls would race on the counter.\nclass TaggedEchoClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"tagged-echo\"\n }\n\n function requested_model(self) -> string {\n \"tagged-echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://tagged-echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n let prompt = input.prompt(ai.internal.build_output_format(input.output_type));\n text_turn(`R${self.calls}<<${prompt.text()}>>`)\n }\n }\n}\n\n// Answers \"R\" to graph prompts and echoes the summary prompts, so the graph descriptions stay predictable.\nclass SummaryEchoClient {\n implements ai.Client {\n function provider(self) -> string {\n \"summary-echo\"\n }\n\n function requested_model(self) -> string {\n \"summary-echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://summary-echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n let text = input.prompt(ai.internal.build_output_format(input.output_type)).text();\n if (text.includes(\"overall summary of a Generalized Additive Model\")) {\n return text_turn(text);\n };\n text_turn(\"R\")\n }\n }\n}\n\ntest \"describe_graph_cot puts the first reply in the second and third turns and the second reply in the third\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = tagged);\n assert.is_true(result.text.starts_with(\"R3<<\"));\n // The third prompt holds R1 (with the first prompt inside it) before R2, and R2 holds R1 too.\n let r1 = position_of(result.text, \"R1<<\");\n let r2 = position_of(result.text, \"R2<<\");\n assert.is_true(r1 > 0);\n assert.is_true(r1 < r2);\n assert.is_true(position_of(result.text.slice(r2, result.text.length()), \"R1<<\") > 0)\n}\n\ntest \"describe_ebm joins the per-feature descriptions with a blank line\" {\n let result = describe_ebm(\n [\"age\", \"bmi\"],\n [\"G1\", \"G2\"],\n \"IMP\",\n 30,\n 7,\n 2,\n default_graph_options(),\n client = SummaryEchoClient {},\n );\n assert.is_true(result.text.includes(\"age: R\\n\\nbmi: R\"))\n}\n\ntest \"answer_benchmark_question puts the reasoning reply in the answer prompt\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = tagged);\n assert.is_true(result.text.starts_with(\"R2<<\"));\n assert.is_true(result.text.includes(\"R1<<\"));\n assert.is_true(\n position_of(result.text, \"R1<<\") < position_of(result.text, \"Summarize your response\"),\n )\n}\n\ntest \"summarize_ebm puts the draft reply in the shorten prompt\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = tagged);\n assert.is_true(result.text.starts_with(\"R2<<\"));\n assert.is_true(position_of(result.text, \"R1<<\") > 0);\n assert.is_true(\n position_of(result.text, \"R1<<\") < position_of(result.text, \"at most 30 sentences\"),\n )\n}\n",
|
|
31
|
+
}
|