ebaml 0.2.3__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ebaml-0.2.3 → ebaml-0.2.4}/PKG-INFO +3 -3
- {ebaml-0.2.3 → ebaml-0.2.4}/README.md +2 -2
- {ebaml-0.2.3 → ebaml-0.2.4}/pyproject.toml +1 -2
- {ebaml-0.2.3 → ebaml-0.2.4}/pyproject.toml.orig +2 -5
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/.baml-generator-output.json +2 -2
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/_baml_sources.py +4 -4
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/_inlinedbaml.py +1 -1
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/llm/options.py +2 -2
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/local/fact_text.py +4 -2
- {ebaml-0.2.3 → ebaml-0.2.4}/LICENSE +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/.baml-generator-output-root +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/.gitignore +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/_function_registry.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/_invocation_types.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/_invocation_types.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/_typemap.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/clients/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/clients/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/content/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/content/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/errors/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/errors/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/events/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/events/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/mcp/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/mcp/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/stream/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/stream/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/tools/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/tools/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/wire/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/ai/wire/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/crypto/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/crypto/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/csv/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/csv/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/env/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/env/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/errors/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/errors/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/fs/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/fs/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/future/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/future/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/glob/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/glob/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/hash/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/hash/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/host/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/host/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/http/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/http/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/io/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/io/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/iter/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/iter/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/json/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/json/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/media/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/media/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/net/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/net/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/ops/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/ops/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/panics/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/panics/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/random/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/random/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/regex/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/regex/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/sap/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/sap/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/spawn/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/spawn/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/sys/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/sys/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/time/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/time/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/toml/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/toml/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/ws/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/ws/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/yaml/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/baml/yaml/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/experimental/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/experimental/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/py.typed +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/class_/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/class_/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/enum/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/enum/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/errors/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/errors/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/function/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/function/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/interface/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/reflect/interface/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/anthropic/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/anthropic/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/anthropic/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/anthropic/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/assert_/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/assert_/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/aws/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/aws/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/aws/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/aws/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/claude_code/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/claude_code/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/claude_code/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/claude_code/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/google/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/google/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/google/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/google/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/openai/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/openai/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/openai/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/openai/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/openai/internal/decisions/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/openai/internal/decisions/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/testing/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/testing/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/trace/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/trace/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/typesafeai/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/typesafeai/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/typesafeai/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/typesafeai/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/vercel/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/vercel/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/vercel/internal/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/baml_sdk/vendor/vercel/internal/__init__.pyi +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/benchmark.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/core/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/core/custom_types.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/core/exceptions.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/graphs/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/graphs/ebm_graph.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/graphs/graph_text.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/graphs/model_kinds.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/graphs/terms.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/llm/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/llm/descriptors.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/llm/provider_errors.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/llm/retry.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/llm/sdk_setup.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/local/__init__.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/local/facts.py +0 -0
- {ebaml-0.2.3 → ebaml-0.2.4}/src/ebaml/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ebaml
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: BAML for Explainable Boosting Machines: talk to your EBM.
|
|
5
5
|
Keywords: baml,llm,explainable-ai,explainable-boosting-machine,interpretml
|
|
6
6
|
Author: Johnny Godoy
|
|
@@ -407,8 +407,8 @@ A field that you set wins over the default of the call. The output limit has two
|
|
|
407
407
|
|
|
408
408
|
| Calls | `max_output_tokens` |
|
|
409
409
|
| --- | --- |
|
|
410
|
-
| Turn 1 of a graph description, the draft of the summary, and the two turns of the benchmark |
|
|
411
|
-
| All other calls, including the local explanations |
|
|
410
|
+
| Turn 1 of a graph description, the draft of the summary, and the two turns of the benchmark | 16000 |
|
|
411
|
+
| All other calls, including the local explanations | 12000 |
|
|
412
412
|
|
|
413
413
|
### Examples
|
|
414
414
|
|
|
@@ -379,8 +379,8 @@ A field that you set wins over the default of the call. The output limit has two
|
|
|
379
379
|
|
|
380
380
|
| Calls | `max_output_tokens` |
|
|
381
381
|
| --- | --- |
|
|
382
|
-
| Turn 1 of a graph description, the draft of the summary, and the two turns of the benchmark |
|
|
383
|
-
| All other calls, including the local explanations |
|
|
382
|
+
| Turn 1 of a graph description, the draft of the summary, and the two turns of the benchmark | 16000 |
|
|
383
|
+
| All other calls, including the local explanations | 12000 |
|
|
384
384
|
|
|
385
385
|
### Examples
|
|
386
386
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ebaml"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.4"
|
|
4
4
|
description = "BAML for Explainable Boosting Machines: talk to your EBM."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -52,7 +52,6 @@ override-dependencies = [
|
|
|
52
52
|
]
|
|
53
53
|
|
|
54
54
|
[tool.uv.exclude-newer-package]
|
|
55
|
-
certifi = false
|
|
56
55
|
baml-bridge = false
|
|
57
56
|
|
|
58
57
|
[tool.uv.build-backend]
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ebaml"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.4"
|
|
4
4
|
description = "BAML for Explainable Boosting Machines: talk to your EBM."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -36,13 +36,10 @@ Changelog = "https://github.com/johnny-godoy/EBaMl/blob/main/CHANGELOG.md"
|
|
|
36
36
|
[tool.uv]
|
|
37
37
|
# A relative span keeps the cutoff 7 days behind today, so uv never picks a release that is less than 7 days old.
|
|
38
38
|
exclude-newer = "7 days"
|
|
39
|
-
# reason: security fixes must not wait for the 7-day cooldown. certifi 2026.7.22 has 1 CVE fixed in 2026.10.19, so Trivy
|
|
40
|
-
# fails on the older version. Remove the entry once its fixed release is more than 7 days old. The check-exclusions
|
|
41
|
-
# hook fails when an entry is no longer needed.
|
|
42
39
|
# reason: baml-bridge must match the BAML toolchain that generates src/baml_sdk, or the SDK fails at start with a version
|
|
43
40
|
# skew error. CI installs the newest toolchain, and toolchain 0.23.0 changed the ai.Client interface, so the repository
|
|
44
41
|
# needs baml-bridge 0.23.0 at once. Remove the entry once 0.23.0 is more than 7 days old.
|
|
45
|
-
exclude-newer-package = {
|
|
42
|
+
exclude-newer-package = { baml-bridge = false }
|
|
46
43
|
# reason: semgrep 1.182.1 pins mcp==1.29.0 and opentelemetry-api~=1.37.0, which have 5 known CVEs fixed in mcp 1.30.0
|
|
47
44
|
# and opentelemetry-api 1.45.1. Dependabot cannot update them, so the override forces the fixed releases. The
|
|
48
45
|
# opentelemetry packages must move together, because semgrep fails to import with opentelemetry-api 1.45 and the older
|
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
},
|
|
16
16
|
{
|
|
17
17
|
"relative_path": "_baml_sources.py",
|
|
18
|
-
"sha256": "
|
|
18
|
+
"sha256": "69373a1c80a74ce0793ddb607b2a97c4f54b1fb6de5dc9526020e4a7a6c9f3b9"
|
|
19
19
|
},
|
|
20
20
|
{
|
|
21
21
|
"relative_path": "_function_registry.py",
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
},
|
|
24
24
|
{
|
|
25
25
|
"relative_path": "_inlinedbaml.py",
|
|
26
|
-
"sha256": "
|
|
26
|
+
"sha256": "e96798567f520fd264e5b42b4b6e350af7b7153c9ee15ab1e425ea215c93b070"
|
|
27
27
|
},
|
|
28
28
|
{
|
|
29
29
|
"relative_path": "_invocation_types.py",
|
|
@@ -24,9 +24,9 @@ from __future__ import annotations
|
|
|
24
24
|
FILES: dict[str, str] = {
|
|
25
25
|
"benchmark_tests.baml": "// Live benchmark tests: a fixed sample of the questions in benchmark/data/, asked through answer_benchmark_question.\n// Each question has one correct answer, so a failure means a wrong answer, not a strict check. The full benchmark\n// (benchmark/run.py) measures the accuracy over every question; these tests only catch a prompt change that breaks\n// the answers. The selector root::Live also matches the testset \"LiveBenchmark\", so offline runs exclude these tests\n// and live runs include them. `baml test` runs from the repository root, and the paths of the data files are relative\n// to it.\n//\n// The sample takes the first questions of each solution in the file, not the questions that a model answers well,\n// because a sample of easy wins cannot catch a regression. The answer formats and the parsers follow `_SPECS` in\n// src/ebaml/benchmark.py. Keep them equal, so that these tests score an answer as the benchmark does.\n\n// The questions of `task` at `indices`, as [question, solution] pairs.\nfunction benchmark_sample(task: string, indices: int[]) -> string[][] {\n let questions = baml.json.decode<string[][]>(baml.fs.read(`benchmark/data/${task}.json`));\n indices.map((i) -> {\n questions[i]\n })\n}\n\n// The letter of the chosen option, as `_parse_choice` reads it: \"a)\" anywhere, or a bare \"a\" at the start.\nfunction benchmark_choice(answer: string) -> string? {\n let lowered = answer.to_lower_case();\n let found = baml.regex.new(\"\\\\b([a-e])\\\\)\").match_once(lowered)\n ?? baml.regex.new(\"^\\\\W*([a-e])\\\\b\").match_once(lowered);\n if (found == null) {\n return null;\n };\n found.groups[1]?.text\n}\n\n// The monotonicity label, as `_parse_monotonicity` reads it. \"not monotone\" goes first, because an answer may say\n// \"not increasing\".\nfunction benchmark_monotonicity(answer: string) -> string? {\n let lowered = answer.to_lower_case();\n for (let label in [\"not monotone\", \"increasing\", \"decreasing\"]) {\n if (lowered.includes(label)) {\n return label;\n }\n }\n ;\n null\n}\n\n// The answer of the model to one question, or \"\" if the model stops without a finished answer, for example when a\n// reasoning model uses up the token limit. The question then counts as wrong, as an answer that the benchmark cannot\n// parse does, and the other questions of the sample still count. Other errors (a missing key, a rate limit) still fail\n// the test, because they say nothing about the answers.\nfunction benchmark_answer(question: string, answer_format: string, client: ai.Client?) -> string {\n answer_benchmark_question(question, answer_format, client = client).text catch (e) {\n ai.errors.FinishReasonError => \"\"\n }\n}\n\n// Asks every question of `sample`, two at a time to stay below the rate limit of the free router, and returns the\n// number of answers that `parse` reads as the solution. `client` serves the offline test; the live tests omit it.\nfunction benchmark_correct(\n sample: string[][],\n answer_format: string,\n parse: (string) -> string?,\n client: ai.Client? = null,\n) -> int {\n let correct = 0;\n let i = 0;\n while (i < sample.length()) {\n let batch = sample.slice(i, i + 2);\n let replies: string[] = await baml.future.all(batch.map((pair) => { spawn { benchmark_answer(pair[0], answer_format, client) } }));\n let j = 0;\n while (j < batch.length()) {\n let answer = replies[j];\n let ok = parse(answer) == parse(batch[j][1]);\n log.info(\n { \"answer\": answer, \"solution\": batch[j][1], \"correct\": ok },\n event_name = \"benchmark_answer\",\n );\n if (ok) {\n correct = correct + 1;\n };\n j = j + 1;\n }\n ;\n i = i + 2;\n }\n ;\n correct\n}\n\ntestset \"LiveBenchmark\" {\n // chance is 2 of 6 (three labels); 4 of 6 lets one or two hard questions fail\n test \"benchmark monotonicity: at least 4 of 6 sample questions are correct\" {\n // 2 \"Decreasing\", 2 \"Increasing\", 2 \"Not monotone\"\n let sample = benchmark_sample(\"monotonicity\", [17, 37, 4, 7, 0, 8]);\n let correct = benchmark_correct(sample, \"'Increasing', 'Decreasing' or 'Not monotone'\", benchmark_monotonicity);\n assert.is_true(correct >= 4)\n }\n\n // chance is about 1 of 6 (five options); 4 of 6 lets one or two hard questions fail\n test \"benchmark function recognition: at least 4 of 6 sample questions are correct\" {\n // the first question of each solution a) to e), and a second c)\n let sample = benchmark_sample(\"function-recognition\", [0, 1, 2, 7, 10, 14]);\n let correct = benchmark_correct(sample, \"the letter of the function, for example 'a)'\", benchmark_choice);\n assert.is_true(correct >= 4)\n }\n}\n\ntestset \"BenchmarkChecks\" {\n test \"benchmark_sample reads the question and the solution at each index\" {\n let sample = benchmark_sample(\"monotonicity\", [17, 4]);\n assert.equal(sample[0][1], \"Decreasing\");\n assert.equal(sample[1][1], \"Increasing\");\n assert.is_true(sample[0][0].includes(\"Think step by step.\"))\n }\n\n test \"benchmark_choice reads the letter like the benchmark scorer\" {\n assert.equal(benchmark_choice(\"**C)**\"), \"c\");\n assert.equal(benchmark_choice(\"(b)\"), \"b\");\n assert.equal(benchmark_choice(\"a\"), \"a\");\n assert.equal(benchmark_choice(\"none of them\"), null)\n }\n\n test \"benchmark_monotonicity reads the label like the benchmark scorer\" {\n assert.equal(benchmark_monotonicity(\"Not monotone.\"), \"not monotone\");\n assert.equal(benchmark_monotonicity(\"Monotone decreasing\"), \"decreasing\");\n assert.equal(benchmark_monotonicity(\"It goes up and down\"), null)\n }\n\n test \"benchmark_correct counts the answers that match the solution\" {\n // the sample holds 2 \"Decreasing\" questions and an odd count, so the last batch has one question\n let sample = benchmark_sample(\"monotonicity\", [17, 4, 0, 37, 7]);\n let correct = benchmark_correct(\n sample,\n \"'Increasing', 'Decreasing' or 'Not monotone'\",\n benchmark_monotonicity,\n client = ConstantClient { answer: \"Decreasing.\" },\n );\n assert.equal(correct, 2)\n }\n test \"benchmark_correct counts an answer that runs out of tokens as wrong\" {\n let sample = benchmark_sample(\"monotonicity\", [17, 37]);\n let correct = benchmark_correct(\n sample,\n \"'Increasing', 'Decreasing' or 'Not monotone'\",\n benchmark_monotonicity,\n client = ExhaustedClient {},\n );\n assert.equal(correct, 0)\n }\n}\n\n// Ends every call at the token limit with no text, as a reasoning model does when its reasoning uses up the limit.\nclass ExhaustedClient {\n implements ai.Client {\n function provider(self) -> string {\n \"exhausted\"\n }\n\n function requested_model(self) -> string {\n \"exhausted\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://exhausted.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n let turn = scripted_turn(\"\", 1, 1);\n turn.stop_reason = ai.content.StopReason.MaxTokens;\n turn\n }\n }\n}\n\n// Answers every call with `answer`, so the score of a sample is the number of questions with that solution.\nclass ConstantClient {\n answer: string,\n\n implements ai.Client {\n function provider(self) -> string {\n \"constant\"\n }\n\n function requested_model(self) -> string {\n \"constant\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://constant.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n scripted_turn(self.answer, 1, 1)\n }\n }\n}\n",
|
|
26
26
|
"local_explanations.baml": "// Local explanations: Python computes the facts of one prediction deterministically (ebaml.local), BAML turns each\n// kind of fact into words. The three kinds are the explainability queries of Barceló et al., \"Model Interpretability\n// through the Lens of Computational Complexity\" (arXiv:2010.12265): minimum change required, sufficient reason and\n// count completions. The global explanation of the model (describe_ebm) is prepended to every prompt.\n\n// Facts of one prediction, one JSON text per kind of explanation.\nclass LocalFactTexts {\n prediction: string,\n counterfactual: string,\n sufficient_reason: string,\n completions: string,\n}\n\nclass LocalExplanationText {\n counterfactual: string,\n sufficient_reason: string,\n completions: string,\n usage: TokenUsage,\n}\n\nfunction local_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You explain individual predictions of a Generalized Additive Model (GAM). You receive an explanation of the whole model and facts about one prediction that were computed exactly from the model. State only effects, values and probabilities that appear in these facts, because the reader acts on your words and an invented number would mislead. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// The audience decides the vocabulary. The facts of a \"layperson\" carry no model numbers, but the instruction repeats it.\nfunction audience_guidance(audience: \"layperson\" | \"data_scientist\") -> string {\n if (audience == \"data_scientist\") {\n return \"The reader is a data scientist, so be precise. Quote the scores (log-odds contributions), probabilities and confidence intervals from the facts. Say how uncertain a statement is when the facts show wide confidence intervals.\";\n };\n \"The reader is not technical and knows no statistics, so write in everyday words. Express strength and likelihood in plain words such as 'strongly', 'slightly', 'very likely' or 'about as likely as not'. Mention only values of the person's own inputs (for example their age), because model numbers (scores, log-odds, probabilities, percentages, intervals, standard deviations) mean nothing to this reader.\"\n}\n\nfunction local_user_msg(\n global_explanation: string,\n prediction: string,\n question: string,\n facts: string,\n audience: \"layperson\" | \"data_scientist\",\n dataset_description: string,\n) -> string {\n let msg = `\n <global_explanation>\n ${global_explanation}\n </global_explanation>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + `\n <prediction>\n ${prediction}\n </prediction>\n\n <facts>\n ${facts}\n </facts>\n\n <audience>\n ${audience_guidance(audience)}\n </audience>\n\n <question>\n ${question}\n </question>\n `\n}\n\nfunction counterfactual_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, what would have to change for the prediction to be different.\n The facts list the smallest set of input changes that flips the prediction (a minimum change required), and whether the flip survives the model's uncertainty.\n This set is exact, not a guess: no smaller set of changes flips the prediction, because the model adds up one independent effect per input.\n Only realistic alternative values were considered (rare extreme values are left out), and the flip holds even if the model's effects are at the unfavourable edge of their confidence interval.\n Suggest only the changes that the facts list, because they are the only ones that the model supports.\n If no change flips the prediction, say that the prediction cannot be changed by altering the inputs.\n `\n}\n\nfunction sufficient_reason_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, which inputs are enough to settle the prediction.\n The facts list a smallest set of inputs (a sufficient reason) that fixes the prediction whatever the other inputs are. State clearly that the remaining inputs cannot change the outcome.\n This is exact: no smaller set of inputs guarantees the outcome, because the model adds up one independent effect per input.\n The guarantee covers every realistic value of the other inputs (rare extreme values are left out) and holds even when the model's effects are at the unfavourable edge of their confidence interval.\n Present it as a statement about this model, not about the real world.\n If there is no such set, say that every input matters.\n `\n}\n\nfunction completions_question(num_sentences: int) -> string {\n `\n Explain, in at most ${num_sentences} sentences, how the outcome gets more certain as the inputs become known.\n The facts give the share of people who would get the same prediction when only the first inputs are known and the rest are unknown (count completions).\n Describe how that share grows and which inputs make the biggest difference.\n Unlike the other two explanations, these shares are estimates from a large random sample that treats the unknown inputs as independent and distributed as in the training data. Describe them as approximate, because they are not guarantees.\n `\n}\n\nfunction LocalCounterfactual(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, counterfactual_question(num_sentences), facts.counterfactual, audience, opts.dataset_description)}\n `\n}\n\nfunction LocalSufficientReason(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, sufficient_reason_question(num_sentences), facts.sufficient_reason, audience, opts.dataset_description)}\n `\n}\n\nfunction LocalCompletions(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${local_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${local_user_msg(global_explanation, facts.prediction, completions_question(num_sentences), facts.completions, audience, opts.dataset_description)}\n `\n}\n\n// Entry point called from Python: the three explanations are independent, so they run concurrently.\nfunction explain_local(\n global_explanation: string,\n facts: LocalFactTexts,\n audience: \"layperson\" | \"data_scientist\",\n num_sentences: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> LocalExplanationText {\n let selected = select_client(client, model, short_defaults());\n let counterfactual = spawn {\n run_counted(\n LocalCounterfactual@spec(global_explanation, facts, audience, num_sentences, opts),\n client = selected,\n )\n };\n let sufficient = spawn {\n run_counted(\n LocalSufficientReason@spec(global_explanation, facts, audience, num_sentences, opts),\n client = selected,\n )\n };\n let completions = spawn {\n run_counted(\n LocalCompletions@spec(global_explanation, facts, audience, num_sentences, opts),\n client = selected,\n )\n };\n let done = await baml.future.all([counterfactual, sufficient, completions]);\n LocalExplanationText {\n counterfactual: done[0].text,\n sufficient_reason: done[1].text,\n completions: done[2].text,\n usage: add_usage(add_usage(done[0].usage, done[1].usage), done[2].usage),\n }\n}\n",
|
|
27
|
-
"model_spec.baml": "// A model chosen at run time. Python sends a `ModelSpec` (plain data) and BAML builds the client, because a client\n// object cannot cross the Python/BAML boundary.\n\nclass ModelSpec {\n provider: string,\n model: string,\n api_key_env: string?,\n base_url: string?,\n temperature: float?,\n max_output_tokens: int?,\n headers: map<string, string>?,\n}\n\n// The settings of one call: used for each field that the `ModelSpec` leaves null.\nclass CallDefaults {\n temperature: float,\n max_output_tokens: int,\n}\n\n// These three must equal the `temperature` and `max_tokens` of `RouterBase`, `RouterLongBase` and\n// `RouterBenchmarkBase` in talk_to_ebm.baml.\nfunction short_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens:
|
|
28
|
-
"model_spec_test.baml": "// Offline tests for model_spec.baml: building a client from a `ModelSpec` sends no request.\n\nfunction spec_for(provider: string) -> ModelSpec {\n ModelSpec {\n provider: provider,\n model: \"m\",\n api_key_env: null,\n base_url: null,\n temperature: null,\n max_output_tokens: null,\n headers: null,\n }\n}\n\ntest \"every provider of the table builds a client with no network access\" {\n let providers = client_builders().keys();\n assert.equal(providers.length(), 9);\n for (let provider in providers) {\n assert.equal(build_client(spec_for(provider), short_defaults()).requested_model(), \"m\");\n }\n}\n\ntest \"the table lists the BAML provider prefixes\" {\n let providers = supported_providers();\n for (\n let prefix in [\n \"openai\",\n \"openai-generic\",\n \"azure\",\n \"openrouter\",\n \"ollama\",\n \"anthropic\",\n \"google\",\n \"vertex\",\n \"bedrock\",\n ]\n ) {\n assert.is_true(providers.includes(prefix));\n }\n}\n\ntest \"a client with every optional field set builds\" {\n let spec = ModelSpec {\n provider: \"openai-generic\",\n model: \"m\",\n api_key_env: \"GATEWAY_KEY\",\n base_url: \"http://gateway.invalid/v1\",\n temperature: 0.2,\n max_output_tokens: 100,\n headers: { \"x-team\": \"ebm\" },\n };\n assert.equal(build_client(spec, short_defaults()).requested_model(), \"m\")\n}\n\ntest \"resolve_settings lets the user values win over the defaults of the call\" {\n let spec = ModelSpec {\n provider: \"openai\",\n model: \"m\",\n api_key_env: null,\n base_url: null,\n temperature: 0.1,\n max_output_tokens: 50,\n headers: null,\n };\n let settings = resolve_settings(spec, long_defaults());\n assert.equal(settings.temperature, 0.1);\n assert.equal(settings.max_output_tokens, 50)\n}\n\ntest \"resolve_settings takes the defaults of the call for null fields\" {\n let settings = resolve_settings(spec_for(\"openai\"), benchmark_defaults());\n assert.equal(settings.temperature, 0.0);\n assert.equal(settings.max_output_tokens,
|
|
29
|
-
"prompt_tests.baml": "// Live prompt tests: real requests to the default Router clients, so they need a real OPENROUTER_API_KEY.\n// They check properties that a good answer must have (no model numbers for a layperson, a sentence limit, the\n// requested answer format), not exact wording, because the free router is not deterministic. Each test logs the reply\n// of the model. If a test fails, read the reply and rerun the test before you change a prompt, because one failure can\n// be noise. Offline runs exclude them: `baml test -x root::Live`.\n\n// Number of sentences in `text`: a run of closing punctuation followed by a space or the end of the text ends a sentence.\n// The period of a decimal (0.62) and the abbreviations \"e.g.\" and \"i.e.\" do not end a sentence.\nfunction count_sentences(text: string) -> int {\n let ends = baml.regex.new(\"[.!?]+[\\\"')\\\\]]*(\\\\s|$)\").match_all(text.trim()).length();\n let abbreviations = baml.regex.new(\"(?i)\\\\b(e\\\\.g|i\\\\.e)\\\\.\\\\s\").match_all(text).length();\n ends - abbreviations\n}\n\ntestset \"PromptChecks\" {\n test \"count_sentences: the period of a decimal does not end a sentence\" {\n assert.equal(count_sentences(\"The debt ratio is 0.62. Income is low.\"), 2)\n }\n\n test \"count_sentences: an abbreviation does not end a sentence\" {\n assert.equal(\n count_sentences(\"Young people, e.g. those under 30, score higher. Older people score lower!\"),\n 2,\n )\n }\n\n test \"count_sentences: a closing quote or bracket after the end counts once\" {\n assert.equal(count_sentences(\"He said \\\"no.\\\" Then (later.) she left...\"), 3)\n }\n\n test \"count_sentences: text with no closing punctuation has no sentence end\" {\n assert.equal(count_sentences(\"Income matters most\"), 0);\n assert.equal(count_sentences(\"\"), 0)\n }\n}\n\n// The facts of one loan prediction, as ebaml.local.fact_text.facts_to_texts renders them for each audience. A layperson\n// gets words in place of the model numbers, so the layperson prompts see no score, probability or share.\nfunction live_layperson_facts() -> LocalFactTexts {\n LocalFactTexts {\n prediction: `{\"predicted_class\": \"loan denied\", \"contributions_largest_first\": [{\"feature\": \"income\", \"value\": \"[1.8e+04, 2.4e+04)\", \"effect\": \"strongly towards this prediction\"}, {\"feature\": \"debt_ratio\", \"value\": \"[0.55, 0.7)\", \"effect\": \"moderately towards this prediction\"}, {\"feature\": \"age\", \"value\": \"[35, 42)\", \"effect\": \"slightly towards this prediction\"}], \"likelihood\": \"very likely\"}`,\n counterfactual: `{\"flip_possible\": true, \"changes\": [{\"feature\": \"income\", \"from\": \"[1.8e+04, 2.4e+04)\", \"to\": \"[4.5e+04, 5e+04)\"}], \"minimum_number_of_changes\": 1, \"flip_holds_even_if_the_model_is_somewhat_off\": true, \"likelihood_of_new_prediction\": \"very unlikely\"}`,\n sufficient_reason: `{\"exists\": true, \"features\": [\"income\", \"debt_ratio\"], \"features_not_needed\": [\"age\"]}`,\n completions: `{\"steps\": [{\"known_features\": [\"income\"], \"share_with_same_prediction\": \"likely\"}, {\"known_features\": [\"income\", \"debt_ratio\"], \"share_with_same_prediction\": \"almost certain\"}]}`,\n }\n}\n\nfunction live_data_scientist_facts() -> LocalFactTexts {\n LocalFactTexts {\n prediction: `{\"predicted_class\": \"loan denied\", \"contributions_largest_first\": [{\"feature\": \"income\", \"value\": \"[1.8e+04, 2.4e+04)\", \"score\": 1.12, \"std\": 0.15}, {\"feature\": \"debt_ratio\", \"value\": \"[0.55, 0.7)\", \"score\": 0.31, \"std\": 0.08}, {\"feature\": \"age\", \"value\": \"[35, 42)\", \"score\": 0.04, \"std\": 0.03}], \"probability\": 0.83, \"confidence_interval\": [0.74, 0.89], \"confidence_level\": 0.95, \"logit\": 1.37}`,\n counterfactual: `{\"flip_possible\": true, \"changes\": [{\"feature\": \"income\", \"from\": \"[1.8e+04, 2.4e+04)\", \"to\": \"[4.5e+04, 5e+04)\", \"score_change\": -2.91}], \"minimum_number_of_changes\": 1, \"flip_holds_at_confidence\": 0.95, \"probability_of_new_prediction\": 0.18}`,\n sufficient_reason: `{\"exists\": true, \"features\": [\"income\", \"debt_ratio\"], \"features_not_needed\": [\"age\"], \"guaranteed_margin_logit\": 0.42}`,\n completions: `{\"steps\": [{\"known_features\": [\"income\"], \"share_with_same_prediction\": 0.71}, {\"known_features\": [\"income\", \"debt_ratio\"], \"share_with_same_prediction\": 1.0}]}`,\n }\n}\n\n// Prints the reply of a live test. An assertion reports only \"expected true\", so the log is how a failed run shows\n// whether the model or the check is wrong.\nfunction log_reply(reply: string) -> null {\n log.info({ \"reply\": reply }, event_name = \"live_reply\")\n}\n\ntestset \"Live\" {\n test \"layperson counterfactual: no model numbers, within the sentence limit, names the input\" {\n let text = LocalCounterfactual(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n assert.is_true(text.includes(\"income\"));\n assert.is_true(!text.includes(\"0.83\") && !text.includes(\"1.37\") && !text.includes(\"2.91\"));\n assert.is_true(!text.includes(\"%\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"layperson sufficient reason: says the other inputs cannot change the outcome\" {\n let text = LocalSufficientReason(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n assert.is_true(text.to_lower_case().includes(\"income\"));\n assert.is_true(text.to_lower_case().includes(\"age\"));\n assert.is_true(!text.includes(\"0.04\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"layperson completions: approximate wording and no percentages\" {\n let text = LocalCompletions(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n assert.is_true(!text.includes(\"71\") && !text.includes(\"%\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"data scientist counterfactual: keeps the figures from the facts\" {\n let text = LocalCounterfactual(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_data_scientist_facts(),\n \"data_scientist\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n // the target bin \"[4.5e+04, 5e+04)\" may be written as \"45,000\", \"45 000\" or \"45k\"\n assert.is_true(baml.regex.new(\"2\\\\.91|4\\\\.5e\\\\+?04|45[,. ]?000|45k\").has_match(text))\n }\n\n test \"explain_local: three explanations, each non-empty, usage reported\" {\n let result = explain_local(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(result.counterfactual);\n log_reply(result.sufficient_reason);\n log_reply(result.completions);\n assert.is_true(result.counterfactual.length() > 0);\n assert.is_true(result.sufficient_reason.length() > 0);\n assert.is_true(result.completions.length() > 0);\n assert.is_true(result.usage.input_tokens > 0);\n assert.is_true(result.usage.output_tokens > 0)\n }\n\n test \"describe_graph_cot: names the feature and respects the sentence limit\" {\n let graph = \"{\\\"feature_name\\\": \\\"age\\\", \\\"type\\\": \\\"continuous\\\", \\\"mean\\\": {\\\"18-30\\\": 0.8, \\\"30-50\\\": 0.0, \\\"50-90\\\": -0.6}}\";\n let result = describe_graph_cot(graph, 3, default_graph_options());\n log_reply(result.text);\n assert.is_true(result.text.to_lower_case().includes(\"age\"));\n assert.is_true(count_sentences(result.text) <= 4);\n assert.is_true(result.usage.input_tokens > 0)\n }\n\n test \"describe_graph_cot: a multiclass graph is read class against class\" {\n // \"young\" leads at low ages and \"old\" at high ages; a good answer names both classes and the change of lead\n let graph = \"Feature Name: age\\nFeature Type: continuous\\nClass: young\\nMeans: {\\\"(18, 30)\\\": 1.5, \\\"(30, 50)\\\": 0.0, \\\"(50, 90)\\\": -1.5}\\nClass: middle\\nMeans: {\\\"(18, 30)\\\": 0.0, \\\"(30, 50)\\\": 0.5, \\\"(50, 90)\\\": 0.0}\\nClass: old\\nMeans: {\\\"(18, 30)\\\": -1.5, \\\"(30, 50)\\\": -0.5, \\\"(50, 90)\\\": 1.5}\";\n let result = describe_graph_cot(graph, 4, default_graph_options());\n log_reply(result.text);\n let text = result.text.to_lower_case();\n assert.is_true(text.includes(\"young\") && text.includes(\"old\"));\n assert.is_true(count_sentences(result.text) <= 5)\n }\n\n test \"describe_graph_cot: a ModelSpec sends the requests to the chosen model\" {\n let graph = \"{\\\"feature_name\\\": \\\"age\\\", \\\"type\\\": \\\"continuous\\\", \\\"mean\\\": {\\\"18-30\\\": 0.8, \\\"30-50\\\": 0.0, \\\"50-90\\\": -0.6}}\";\n let model = ModelSpec {\n provider: \"openrouter\",\n model: \"openrouter/free\",\n api_key_env: \"OPENROUTER_API_KEY\",\n base_url: null,\n temperature: null,\n max_output_tokens: null,\n headers: null,\n };\n let result = describe_graph_cot(graph, 3, default_graph_options(), model = model);\n log_reply(result.text);\n assert.is_true(result.text.to_lower_case().includes(\"age\"));\n assert.is_true(result.usage.output_tokens > 0)\n }\n\n test \"answer_benchmark_question: the final turn follows the requested format\" {\n let question = \"A model's graph for feature x has mean value 2.0 for x < 5 and -1.0 for x >= 5. Is the function decreasing in x? Think step by step.\";\n let result = answer_benchmark_question(question, \"'yes' or 'no'\");\n log_reply(result.text);\n let answer = result.text.trim().to_lower_case();\n assert.is_true(answer.includes(\"yes\"));\n assert.is_true(answer.length() < 20)\n }\n}\n",
|
|
30
|
-
"talk_to_ebm.baml": "// Port of TalkToEBM's t2ebm/prompts.py and t2ebm/llm.py: prompts, chat clients, CoT pipelines, token accounting.\n\n// Some models behind `openrouter/free` reason mandatorily (reasoning cannot be disabled), and the hidden reasoning tokens\n// count against `max_tokens`, so the caps are generous.\n// Transient failures (rate limits, network, 5xx) are retried; rejected requests are not.\nclient RouterBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 6000);\n\n// First assistant turn of each conversation allows a longer answer.\nclient RouterLongBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 8000);\n\nclient Router = ai.clients.Retry.new(inner = RouterBase, max_attempts = 3);\n\n// The benchmark is scored on one answer per question, so it samples greedily, as TalkToEBM's benchmark does.\nclient RouterBenchmarkBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.0, max_tokens = 8000);\n\nclient RouterBenchmark = ai.clients.Retry.new(inner = RouterBenchmarkBase, max_attempts = 3);\n\nclient RouterLong = ai.clients.Retry.new(inner = RouterLongBase, max_attempts = 3);\n\nclass GraphPromptOptions {\n expert_description: string,\n graph_description: string,\n dataset_description: string,\n task_description: string,\n}\n\nfunction default_graph_options() -> GraphPromptOptions {\n GraphPromptOptions {\n expert_description: \"an expert statistician and data scientist\",\n graph_description: \"\",\n dataset_description: \"\",\n task_description: \"Describe the general pattern of the graph.\",\n }\n}\n\n// Prompt text helpers. Pure functions, so the Jinja `{% if %}` blocks become plain BAML.\n\nfunction graph_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You interpret global explanations produced by a Generalized Additive Model (GAM). Base every answer on the graph from the user, on any other information you are given, and on your knowledge of the real world. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// How to read the series of the classes. Only the graph of a multiclass classifier has \"Class: \" lines, so the prompt\n// of a binary classifier or a regressor gets no text about classes.\nfunction multiclass_format(graph: string) -> string {\n if (graph.includes(\"Class: \")) {\n return \"\\nA graph of a multiclass classifier repeats items 3 to 5 once for each class, after a line \\\"Class: <name>\\\". In every bin, the scores have the same mean over all the classes of the model, including classes that the graph leaves out. Only the differences between the classes change the predicted probabilities, so compare the classes with each other within each bin.\";\n };\n \"\"\n}\n\nfunction describe_graph_msg(graph: string, opts: GraphPromptOptions) -> string {\n let msg = `\n <graph_format>\n The graph is a JSON object. Its keys are the x-axis and its values are the y-axis. For a continuous feature, each key is an interval where the function predicts the same value. For a categorical feature, each key is a possible value of the feature.\n The graph contains these items:\n 1. The name of the feature.\n 2. The type of the feature (continuous, categorical, or boolean).\n 3. The mean values.\n 4. The lower bounds of the confidence interval (optional).\n 5. The upper bounds of the confidence interval (optional).${multiclass_format(graph)}\n </graph_format>\n\n <graph>\n ${graph}\n </graph>\n `\n + \"\\n\\n\";\n if (opts.graph_description.length() > 0) {\n msg = msg + `<graph_description>\\n${opts.graph_description}\\n</graph_description>\\n\\n`;\n };\n if (opts.dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${opts.dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg + `<task>\\n${opts.task_description}\\n</task>`\n}\n\nfunction summarize_ebm_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You provide an overall summary of a Generalized Additive Model (GAM). The model consists of different graphs, and each graph shows the effect of one input feature. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\nfunction summarize_ebm_msg(\n feature_importances: string,\n graph_descriptions: string,\n dataset_description: string,\n) -> string {\n let msg = `\n <feature_importances>\n ${feature_importances}\n </feature_importances>\n\n <graph_descriptions>\n ${graph_descriptions}\n </graph_descriptions>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + `\n <instructions>\n Summarize the model. The feature importances are global, and there is exactly one graph description for each feature.\n Cover the most important features and their effect on the outcome. Leave out unimportant features and effects, so the summary stays short.\n Include any surprising pattern, because the reader wants to know where the model differs from expectation.\n </instructions>\n `\n}\n\n// Request text shared by the second turn and the replay of that turn in the third.\nfunction surprises_request() -> string {\n `\n <instructions>\n Study the graph carefully. Highlight any region that you find surprising or counterintuitive, and name its range or category so the reader can find it. Suggest why the behavior is surprising and what may have caused it.\n </instructions>\n `\n}\n\nfunction final_description_request(num_sentences: int) -> string {\n `\n <instructions>\n Describe the graph in at most ${num_sentences} sentences.\n Include the important surprising patterns, because the reader uses them to decide where to look closer.\n The reader knows that the graph is from a Generalized Additive Model (GAM), so leave that background out.\n </instructions>\n `\n}\n\n// describe_graph_cot, turn 1: system + describe_graph prompt.\nfunction GraphFirstPass(graph: string, opts: GraphPromptOptions) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n `\n}\n\n// Turn 2: surprising regions. `first_pass` is the assistant reply from turn 1.\nfunction GraphSurprises(graph: string, first_pass: string, opts: GraphPromptOptions) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n `\n}\n\n// Turn 3: brief final description.\nfunction GraphFinalDescription(\n graph: string,\n first_pass: string,\n surprises: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n ${role(\"assistant\")}\n ${surprises}\n ${role(\"user\")}\n ${final_description_request(num_sentences)}\n `\n}\n\n// summarize_ebm, turn 1.\nfunction EbmSummaryDraft(\n feature_importances: string,\n graph_descriptions: string,\n opts: GraphPromptOptions,\n) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description)}\n `\n}\n\n// summarize_ebm, turn 2: shorten the draft.\nfunction EbmSummaryShorten(\n feature_importances: string,\n graph_descriptions: string,\n draft: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description)}\n ${role(\"assistant\")}\n ${draft}\n ${role(\"user\")}\n <instructions>\n Shorten your summary to at most ${num_sentences} sentences. Keep the most important information, because the reader wants the main effects and surprises.\n </instructions>\n `\n}\n\n// Token accounting.\n\nclass TokenUsage {\n input_tokens: int,\n output_tokens: int,\n}\n\n// An LLM result plus the tokens the provider reported for producing it.\nclass Counted {\n text: string,\n usage: TokenUsage,\n}\n\nclass TokenBudget {\n estimated_tokens: int,\n status: \"ok\" | \"warn\" | \"over\",\n}\n\nfunction add_usage(a: TokenUsage, b: TokenUsage) -> TokenUsage {\n TokenUsage {\n input_tokens: a.input_tokens + b.input_tokens,\n output_tokens: a.output_tokens + b.output_tokens,\n }\n}\n\n// Rough estimate (~3 chars/token): graph text is mostly digits and punctuation, which tokenize worse than prose.\n// No tokenizer is available before a call, so this is a guard rail; the provider's own error is the real limit.\nfunction estimate_tokens(text: string) -> int {\n (text.length() + 2) / 3\n}\n\n// \"ok\" within `max_tokens`; \"warn\" up to `max_tokens` plus `overbudget_percent`; \"over\" beyond that.\nfunction check_token_budget(text: string, max_tokens: int, overbudget_percent: int) -> TokenBudget {\n let estimated = estimate_tokens(text);\n let hard_limit = max_tokens * (100 + overbudget_percent) / 100;\n if (estimated > hard_limit) {\n return TokenBudget { estimated_tokens: estimated, status: \"over\" };\n };\n if (estimated > max_tokens) {\n return TokenBudget { estimated_tokens: estimated, status: \"warn\" };\n };\n TokenBudget { estimated_tokens: estimated, status: \"ok\" }\n}\n\n// Runs one text-producing LLM function and keeps the provider's token usage. A count that the provider did not report\n// is zero.\nfunction run_counted(spec: ai.FunctionSpec<string>, client: ai.Client? = null) -> Counted {\n let result = ai.Agent.new(client = client).run(spec);\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0 };\n for (let entry in result.usage) {\n usage\n = add_usage(\n usage,\n TokenUsage {\n input_tokens: entry.usage.input_tokens ?? 0,\n output_tokens: entry.usage.output_tokens ?? 0,\n },\n );\n }\n Counted { text: result.value, usage: usage }\n}\n\n// Entry points called from Python.\n\nfunction describe_graph_cot(\n graph: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n //# Three sequential assistant turns, each seeing the previous replies\n let long = select_client(client, model, long_defaults());\n let short = select_client(client, model, short_defaults());\n let first_pass = run_counted(GraphFirstPass@spec(graph, opts), client = long);\n let surprises = run_counted(GraphSurprises@spec(graph, first_pass.text, opts), client = short);\n let final = run_counted(\n GraphFinalDescription@spec(graph, first_pass.text, surprises.text, num_sentences, opts),\n client = short,\n );\n Counted {\n text: final.text,\n usage: add_usage(add_usage(first_pass.usage, surprises.usage), final.usage),\n }\n}\n\n// `num_sentences` null keeps the draft summary as is.\nfunction summarize_ebm(\n feature_importances: string,\n graph_descriptions: string,\n num_sentences: int?,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let draft = run_counted(\n EbmSummaryDraft@spec(feature_importances, graph_descriptions, opts),\n client = select_client(client, model, long_defaults()),\n );\n if (num_sentences == null) {\n return draft;\n };\n let shortened = run_counted(\n EbmSummaryShorten@spec(feature_importances, graph_descriptions, draft.text, num_sentences, opts),\n client = select_client(client, model, short_defaults()),\n );\n Counted { text: shortened.text, usage: add_usage(draft.usage, shortened.usage) }\n}\n\n// Full EBM description: one chain per feature graph (concurrently), then the summary.\n// `graphs[i]` is the text of the graph of `feature_names[i]`.\nfunction describe_ebm(\n feature_names: string[],\n graphs: string[],\n feature_importances: string,\n num_sentences: int,\n graph_sentences: int,\n max_concurrency: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n if (feature_names.length() != graphs.length()) {\n throw baml.errors.InvalidArgument { message: \"feature_names and graphs must have the same length\" };\n };\n //# One description chain per graph, at most `max_concurrency` at a time\n let limit = baml.spawn.Limit.new(max_concurrency);\n let futures = graphs.map((graph) -> {\n spawn with limit {\n describe_graph_cot(graph, graph_sentences, opts, client = client, model = model)\n }\n });\n let described: Counted[] = await baml.future.all(futures);\n\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0 };\n let lines: string[] = [];\n let i = 0;\n while (i < described.length()) {\n let item = described[i];\n usage = add_usage(usage, item.usage);\n lines.push(`${feature_names[i]}: ${item.text}`);\n i = i + 1;\n }\n //# Summarize all the per-graph descriptions\n let summary = summarize_ebm(feature_importances, lines.join(\"\\n\\n\"), num_sentences, opts, client = client, model = model);\n Counted { text: summary.text, usage: add_usage(usage, summary.usage) }\n}\n\n// Benchmark (benchmark/): the questions end with \"Think step by step.\", so turn 1 reasons freely and turn 2\n// extracts the final answer in the format the scorer parses.\n\nfunction BenchmarkReasoning(question: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n `\n}\n\nfunction BenchmarkAnswer(question: string, reasoning: string, answer_format: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n ${role(\"assistant\")}\n ${reasoning}\n ${role(\"user\")}\n <instructions>\n Summarize your response by answering with ${answer_format}. Reply with the answer only, because a program parses your reply.\n </instructions>\n `\n}\n\nfunction answer_benchmark_question(\n question: string,\n answer_format: string,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let selected = select_client(client, model, benchmark_defaults());\n let reasoning = run_counted(BenchmarkReasoning@spec(question), client = selected);\n let answer = run_counted(BenchmarkAnswer@spec(question, reasoning.text, answer_format), client = selected);\n Counted { text: answer.text, usage: add_usage(reasoning.usage, answer.usage) }\n}\n",
|
|
27
|
+
"model_spec.baml": "// A model chosen at run time. Python sends a `ModelSpec` (plain data) and BAML builds the client, because a client\n// object cannot cross the Python/BAML boundary.\n\nclass ModelSpec {\n provider: string,\n model: string,\n api_key_env: string?,\n base_url: string?,\n temperature: float?,\n max_output_tokens: int?,\n headers: map<string, string>?,\n}\n\n// The settings of one call: used for each field that the `ModelSpec` leaves null.\nclass CallDefaults {\n temperature: float,\n max_output_tokens: int,\n}\n\n// These three must equal the `temperature` and `max_tokens` of `RouterBase`, `RouterLongBase` and\n// `RouterBenchmarkBase` in talk_to_ebm.baml.\nfunction short_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens: 12000 }\n}\n\n// The first turn of each conversation allows a longer answer.\nfunction long_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.7, max_output_tokens: 16000 }\n}\n\nfunction benchmark_defaults() -> CallDefaults {\n CallDefaults { temperature: 0.0, max_output_tokens: 16000 }\n}\n\n// A field that the user set wins. Otherwise the default of the call applies.\nfunction resolve_settings(spec: ModelSpec, defaults: CallDefaults) -> CallDefaults {\n CallDefaults {\n temperature: spec.temperature ?? defaults.temperature,\n max_output_tokens: spec.max_output_tokens ?? defaults.max_output_tokens,\n }\n}\n\n// The key is read from the environment when the request is sent. Null selects the default variable of the provider.\nfunction credential(name: string?) -> ai.Credential? {\n if (name == null) {\n return null;\n };\n baml.env.ref(name)\n}\n\nfunction build_openai(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.ChatClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_openai_generic(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.GenericClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// Azure serves deployments under names that the owner chose, so `model` names the deployment. `base_url` names it too\n// and wins. Without it, the deployment is built on the endpoint in `AZURE_OPENAI_ENDPOINT`.\nfunction build_azure(spec: ModelSpec, settings: CallDefaults) -> openai.AzureClient {\n openai.AzureClient.new(\n model = spec.model,\n deployment_id = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_openrouter(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.OpenRouterClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_ollama(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n openai.OllamaClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_anthropic(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n anthropic.Client.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// Gemini names its output limit `max_output_tokens`.\nfunction build_google(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n google.GeminiClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_output_tokens = settings.max_output_tokens,\n )\n}\n\nfunction build_vertex(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n google.VertexClient.new(\n model = spec.model,\n api_key = credential(spec.api_key_env),\n base_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_output_tokens = settings.max_output_tokens,\n )\n}\n\n// Bedrock signs requests with AWS credentials, so `api_key_env` has no effect.\nfunction build_bedrock(spec: ModelSpec, settings: CallDefaults) -> ai.Client {\n aws.BedrockClient.new(\n model = spec.model,\n endpoint_url = spec.base_url,\n headers = spec.headers,\n temperature = settings.temperature,\n max_tokens = settings.max_output_tokens,\n )\n}\n\n// One row for each provider. A new provider is a new row. The keys are the BAML provider prefixes.\nfunction client_builders() -> map<string, (ModelSpec, CallDefaults) -> ai.Client throws never> {\n {\n \"openai\": build_openai,\n \"openai-generic\": build_openai_generic,\n \"azure\": build_azure,\n \"openrouter\": build_openrouter,\n \"ollama\": build_ollama,\n \"anthropic\": build_anthropic,\n \"google\": build_google,\n \"vertex\": build_vertex,\n \"bedrock\": build_bedrock,\n }\n}\n\n// The provider prefixes that `ModelSpec.provider` accepts, in table order.\nfunction supported_providers() -> string[] {\n client_builders().keys()\n}\n\n// Builds the client of `spec` with the retry policy of the default clients.\nfunction build_client(spec: ModelSpec, defaults: CallDefaults) -> ai.Client {\n let builders = client_builders();\n let build = builders.get(spec.provider);\n if (build == null) {\n throw baml.errors.InvalidArgument {\n message: `unknown provider \"${spec.provider}\"; supported providers: ${supported_providers().join(\", \")}`,\n };\n };\n ai.clients.Retry.new(inner = build(spec, resolve_settings(spec, defaults)), max_attempts = 3)\n}\n\n// Builds and discards the client, so that Python can reject a bad provider before the first paid call.\nfunction check_model(spec: ModelSpec) -> bool {\n build_client(spec, short_defaults());\n true\n}\n\n// Precedence: `client` (the test hook) > `model` > null, which selects the client declared by the function.\nfunction select_client(\n client: ai.Client?,\n model: ModelSpec?,\n defaults: CallDefaults,\n) -> ai.Client? {\n if (client != null) {\n return client;\n };\n if (model == null) {\n return null;\n };\n build_client(model, defaults)\n}\n",
|
|
28
|
+
"model_spec_test.baml": "// Offline tests for model_spec.baml: building a client from a `ModelSpec` sends no request.\n\nfunction spec_for(provider: string) -> ModelSpec {\n ModelSpec {\n provider: provider,\n model: \"m\",\n api_key_env: null,\n base_url: null,\n temperature: null,\n max_output_tokens: null,\n headers: null,\n }\n}\n\ntest \"every provider of the table builds a client with no network access\" {\n let providers = client_builders().keys();\n assert.equal(providers.length(), 9);\n for (let provider in providers) {\n assert.equal(build_client(spec_for(provider), short_defaults()).requested_model(), \"m\");\n }\n}\n\ntest \"the table lists the BAML provider prefixes\" {\n let providers = supported_providers();\n for (\n let prefix in [\n \"openai\",\n \"openai-generic\",\n \"azure\",\n \"openrouter\",\n \"ollama\",\n \"anthropic\",\n \"google\",\n \"vertex\",\n \"bedrock\",\n ]\n ) {\n assert.is_true(providers.includes(prefix));\n }\n}\n\ntest \"a client with every optional field set builds\" {\n let spec = ModelSpec {\n provider: \"openai-generic\",\n model: \"m\",\n api_key_env: \"GATEWAY_KEY\",\n base_url: \"http://gateway.invalid/v1\",\n temperature: 0.2,\n max_output_tokens: 100,\n headers: { \"x-team\": \"ebm\" },\n };\n assert.equal(build_client(spec, short_defaults()).requested_model(), \"m\")\n}\n\ntest \"resolve_settings lets the user values win over the defaults of the call\" {\n let spec = ModelSpec {\n provider: \"openai\",\n model: \"m\",\n api_key_env: null,\n base_url: null,\n temperature: 0.1,\n max_output_tokens: 50,\n headers: null,\n };\n let settings = resolve_settings(spec, long_defaults());\n assert.equal(settings.temperature, 0.1);\n assert.equal(settings.max_output_tokens, 50)\n}\n\ntest \"resolve_settings takes the defaults of the call for null fields\" {\n let settings = resolve_settings(spec_for(\"openai\"), benchmark_defaults());\n assert.equal(settings.temperature, 0.0);\n assert.equal(settings.max_output_tokens, 16000)\n}\n\n// A reasoning model counts its hidden reasoning in the output limit. At 8000 tokens, live runs ended at the limit with\n// no answer, so the limits are 12000 and 16000.\ntest \"the call defaults differ by call as the declared clients do\" {\n assert.equal(short_defaults().max_output_tokens, 12000);\n assert.equal(long_defaults().max_output_tokens, 16000);\n assert.equal(benchmark_defaults().max_output_tokens, 16000);\n assert.equal(short_defaults().temperature, 0.7);\n assert.equal(benchmark_defaults().temperature, 0.0)\n}\n\n// The message of the error that `build_client` throws, or \"built\" if it throws none.\nfunction failure_message(provider: string) -> string {\n let built = build_client(spec_for(provider), short_defaults()) catch (e) {\n baml.errors.InvalidArgument => {\n return e.message;\n },\n };\n \"built\"\n}\n\n// The provider and the model that a client asks for, or \"null\".\nfunction client_id(client: ai.Client?) -> string {\n if (client == null) {\n return \"null\";\n };\n `${client.provider()}/${client.requested_model()}`\n}\n\ntest \"an unknown provider throws InvalidArgument that lists the supported providers\" {\n let message = failure_message(\"nope\");\n assert.is_true(message.includes(\"nope\"));\n assert.is_true(message.includes(\"anthropic\"));\n assert.is_true(message.includes(\"openai-generic\"))\n}\n\ntest \"a known provider throws no error\" {\n assert.equal(failure_message(\"ollama\"), \"built\")\n}\n\ntest \"check_model accepts a known provider and rejects an unknown one\" {\n assert.is_true(check_model(spec_for(\"ollama\")));\n let outcome = check_model(spec_for(\"nope\")) catch (e) {\n baml.errors.InvalidArgument => false,\n };\n assert.equal(outcome, false)\n}\n\ntest \"select_client prefers the client over the model\" {\n let scripted = ScriptedClient { calls: 0 };\n let selected = select_client(scripted, spec_for(\"ollama\"), short_defaults());\n assert.equal(client_id(selected), \"scripted/scripted\")\n}\n\ntest \"select_client builds the client of the model\" {\n let selected = select_client(null, spec_for(\"ollama\"), short_defaults());\n assert.equal(client_id(selected), \"ollama/m\")\n}\n\ntest \"select_client returns null with no client and no model, so the declared client runs\" {\n assert.equal(client_id(select_client(null, null, short_defaults())), \"null\")\n}\n\ntest \"describe_graph_cot sends every turn to the client when both client and model are set\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_graph_cot(\"G\", 7, default_graph_options(), client = scripted, model = spec_for(\"ollama\"));\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 3)\n}\n\ntest \"an Azure client without base_url names the deployment after the model\" {\n let azure = build_azure(spec_for(\"azure\"), short_defaults());\n assert.equal(azure.deployment_id, \"m\")\n}\n",
|
|
29
|
+
"prompt_tests.baml": "// Live prompt tests: real requests to the default Router clients, so they need a real OPENROUTER_API_KEY.\n// They check properties that a good answer must have (no model numbers for a layperson, a sentence limit, the\n// requested answer format), not exact wording, because the free router is not deterministic. Each test logs the reply\n// of the model. If a test fails, read the reply and rerun the test before you change a prompt, because one failure can\n// be noise. Offline runs exclude them: `baml test -x root::Live`.\n\n// Number of sentences in `text`: a run of closing punctuation followed by a space or the end of the text ends a sentence.\n// The period of a decimal (0.62) and the abbreviations \"e.g.\" and \"i.e.\" do not end a sentence.\nfunction count_sentences(text: string) -> int {\n let ends = baml.regex.new(\"[.!?]+[\\\"')\\\\]]*(\\\\s|$)\").match_all(text.trim()).length();\n let abbreviations = baml.regex.new(\"(?i)\\\\b(e\\\\.g|i\\\\.e)\\\\.\\\\s\").match_all(text).length();\n ends - abbreviations\n}\n\ntestset \"PromptChecks\" {\n test \"count_sentences: the period of a decimal does not end a sentence\" {\n assert.equal(count_sentences(\"The debt ratio is 0.62. Income is low.\"), 2)\n }\n\n test \"count_sentences: an abbreviation does not end a sentence\" {\n assert.equal(\n count_sentences(\"Young people, e.g. those under 30, score higher. Older people score lower!\"),\n 2,\n )\n }\n\n test \"count_sentences: a closing quote or bracket after the end counts once\" {\n assert.equal(count_sentences(\"He said \\\"no.\\\" Then (later.) she left...\"), 3)\n }\n\n test \"count_sentences: text with no closing punctuation has no sentence end\" {\n assert.equal(count_sentences(\"Income matters most\"), 0);\n assert.equal(count_sentences(\"\"), 0)\n }\n}\n\n// The facts of one loan prediction, as ebaml.local.fact_text.facts_to_texts renders them for each audience. A layperson\n// gets words in place of the model numbers, so the layperson prompts see no score, probability or share.\nfunction live_layperson_facts() -> LocalFactTexts {\n LocalFactTexts {\n prediction: `{\"predicted_class\": \"loan denied\", \"contributions_largest_first\": [{\"feature\": \"income\", \"value\": \"[1.8e+04, 2.4e+04)\", \"effect\": \"strongly towards this prediction\"}, {\"feature\": \"debt_ratio\", \"value\": \"[0.55, 0.7)\", \"effect\": \"moderately towards this prediction\"}, {\"feature\": \"age\", \"value\": \"[35, 42)\", \"effect\": \"slightly towards this prediction\"}], \"likelihood\": \"very likely\"}`,\n counterfactual: `{\"flip_possible\": true, \"changes\": [{\"feature\": \"income\", \"from\": \"[1.8e+04, 2.4e+04)\", \"to\": \"[4.5e+04, 5e+04)\"}], \"minimum_number_of_changes\": 1, \"new_predicted_class\": \"loan approved\", \"flip_holds_even_if_the_model_is_somewhat_off\": true, \"likelihood_of_new_predicted_class\": \"very likely\"}`,\n sufficient_reason: `{\"exists\": true, \"features\": [\"income\", \"debt_ratio\"], \"features_not_needed\": [\"age\"]}`,\n completions: `{\"steps\": [{\"known_features\": [\"income\"], \"share_with_same_prediction\": \"likely\"}, {\"known_features\": [\"income\", \"debt_ratio\"], \"share_with_same_prediction\": \"almost certain\"}]}`,\n }\n}\n\nfunction live_data_scientist_facts() -> LocalFactTexts {\n LocalFactTexts {\n prediction: `{\"predicted_class\": \"loan denied\", \"contributions_largest_first\": [{\"feature\": \"income\", \"value\": \"[1.8e+04, 2.4e+04)\", \"score\": 1.12, \"std\": 0.15}, {\"feature\": \"debt_ratio\", \"value\": \"[0.55, 0.7)\", \"score\": 0.31, \"std\": 0.08}, {\"feature\": \"age\", \"value\": \"[35, 42)\", \"score\": 0.04, \"std\": 0.03}], \"probability\": 0.83, \"confidence_interval\": [0.74, 0.89], \"confidence_level\": 0.95, \"logit\": 1.37}`,\n counterfactual: `{\"flip_possible\": true, \"changes\": [{\"feature\": \"income\", \"from\": \"[1.8e+04, 2.4e+04)\", \"to\": \"[4.5e+04, 5e+04)\", \"score_change\": -2.91}], \"minimum_number_of_changes\": 1, \"new_predicted_class\": \"loan approved\", \"flip_holds_at_confidence\": 0.95, \"probability_of_new_predicted_class\": 0.82}`,\n sufficient_reason: `{\"exists\": true, \"features\": [\"income\", \"debt_ratio\"], \"features_not_needed\": [\"age\"], \"guaranteed_margin_logit\": 0.42}`,\n completions: `{\"steps\": [{\"known_features\": [\"income\"], \"share_with_same_prediction\": 0.71}, {\"known_features\": [\"income\", \"debt_ratio\"], \"share_with_same_prediction\": 1.0}]}`,\n }\n}\n\n// Prints the reply of a live test. An assertion reports only \"expected true\", so the log is how a failed run shows\n// whether the model or the check is wrong.\nfunction log_reply(reply: string) -> null {\n log.info({ \"reply\": reply }, event_name = \"live_reply\")\n}\n\ntestset \"Live\" {\n test \"layperson counterfactual: no model numbers, within the sentence limit, names the input\" {\n let text = LocalCounterfactual(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n assert.is_true(text.includes(\"income\"));\n assert.is_true(!text.includes(\"0.83\") && !text.includes(\"1.37\") && !text.includes(\"2.91\"));\n assert.is_true(!text.includes(\"%\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"layperson sufficient reason: says the other inputs cannot change the outcome\" {\n let text = LocalSufficientReason(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n assert.is_true(text.to_lower_case().includes(\"income\"));\n assert.is_true(text.to_lower_case().includes(\"age\"));\n assert.is_true(!text.includes(\"0.04\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"layperson completions: approximate wording and no percentages\" {\n let text = LocalCompletions(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n assert.is_true(!text.includes(\"71\") && !text.includes(\"%\"));\n assert.is_true(count_sentences(text) <= 3)\n }\n\n test \"data scientist counterfactual: keeps the figures from the facts\" {\n let text = LocalCounterfactual(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_data_scientist_facts(),\n \"data_scientist\",\n 3,\n default_graph_options(),\n );\n log_reply(text);\n // the target bin \"[4.5e+04, 5e+04)\" may be written as \"45,000\", \"45 000\" or \"45k\"\n assert.is_true(baml.regex.new(\"2\\\\.91|4\\\\.5e\\\\+?04|45[,. ]?000|45k\").has_match(text))\n }\n\n test \"explain_local: three explanations, each non-empty, usage reported\" {\n let result = explain_local(\n \"The model denies loans mainly because of low income and a high debt ratio.\",\n live_layperson_facts(),\n \"layperson\",\n 3,\n default_graph_options(),\n );\n log_reply(result.counterfactual);\n log_reply(result.sufficient_reason);\n log_reply(result.completions);\n assert.is_true(result.counterfactual.length() > 0);\n assert.is_true(result.sufficient_reason.length() > 0);\n assert.is_true(result.completions.length() > 0);\n assert.is_true(result.usage.input_tokens > 0);\n assert.is_true(result.usage.output_tokens > 0)\n }\n\n test \"describe_graph_cot: names the feature and respects the sentence limit\" {\n let graph = \"{\\\"feature_name\\\": \\\"age\\\", \\\"type\\\": \\\"continuous\\\", \\\"mean\\\": {\\\"18-30\\\": 0.8, \\\"30-50\\\": 0.0, \\\"50-90\\\": -0.6}}\";\n let result = describe_graph_cot(graph, 3, default_graph_options());\n log_reply(result.text);\n assert.is_true(result.text.to_lower_case().includes(\"age\"));\n assert.is_true(count_sentences(result.text) <= 4);\n assert.is_true(result.usage.input_tokens > 0)\n }\n\n test \"describe_graph_cot: a multiclass graph is read class against class\" {\n // \"young\" leads at low ages and \"old\" at high ages; a good answer names both classes and the change of lead\n let graph = \"Feature Name: age\\nFeature Type: continuous\\nClass: young\\nMeans: {\\\"(18, 30)\\\": 1.5, \\\"(30, 50)\\\": 0.0, \\\"(50, 90)\\\": -1.5}\\nClass: middle\\nMeans: {\\\"(18, 30)\\\": 0.0, \\\"(30, 50)\\\": 0.5, \\\"(50, 90)\\\": 0.0}\\nClass: old\\nMeans: {\\\"(18, 30)\\\": -1.5, \\\"(30, 50)\\\": -0.5, \\\"(50, 90)\\\": 1.5}\";\n let result = describe_graph_cot(graph, 4, default_graph_options());\n log_reply(result.text);\n let text = result.text.to_lower_case();\n assert.is_true(text.includes(\"young\") && text.includes(\"old\"));\n assert.is_true(count_sentences(result.text) <= 5)\n }\n\n test \"describe_graph_cot: a ModelSpec sends the requests to the chosen model\" {\n let graph = \"{\\\"feature_name\\\": \\\"age\\\", \\\"type\\\": \\\"continuous\\\", \\\"mean\\\": {\\\"18-30\\\": 0.8, \\\"30-50\\\": 0.0, \\\"50-90\\\": -0.6}}\";\n let model = ModelSpec {\n provider: \"openrouter\",\n model: \"openrouter/free\",\n api_key_env: \"OPENROUTER_API_KEY\",\n base_url: null,\n temperature: null,\n max_output_tokens: null,\n headers: null,\n };\n let result = describe_graph_cot(graph, 3, default_graph_options(), model = model);\n log_reply(result.text);\n assert.is_true(result.text.to_lower_case().includes(\"age\"));\n assert.is_true(result.usage.output_tokens > 0)\n }\n\n test \"answer_benchmark_question: the final turn follows the requested format\" {\n let question = \"A model's graph for feature x has mean value 2.0 for x < 5 and -1.0 for x >= 5. Is the function decreasing in x? Think step by step.\";\n let result = answer_benchmark_question(question, \"'yes' or 'no'\");\n log_reply(result.text);\n // The step function is decreasing but not strictly, so a careful model may answer either way. The LiveBenchmark\n // tests check the answers; this test checks only that the final turn is the bare answer.\n let answer = result.text.trim().to_lower_case();\n assert.is_true(baml.regex.new(\"^\\\\W*(yes|no)\\\\W*$\").has_match(answer))\n }\n}\n",
|
|
30
|
+
"talk_to_ebm.baml": "// Port of TalkToEBM's t2ebm/prompts.py and t2ebm/llm.py: prompts, chat clients, CoT pipelines, token accounting.\n\n// Some models behind `openrouter/free` reason mandatorily (reasoning cannot be disabled), and the hidden reasoning tokens\n// count against `max_tokens`, so the caps are generous. 16000 stays below the output cap of 16384 tokens of common OpenAI\n// models, because these limits are also the defaults of a ModelSpec for any provider.\n// Transient failures (rate limits, network, 5xx) are retried; rejected requests are not.\nclient RouterBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 12000);\n\n// First assistant turn of each conversation allows a longer answer.\nclient RouterLongBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.7, max_tokens = 16000);\n\nclient Router = ai.clients.Retry.new(inner = RouterBase, max_attempts = 3);\n\n// The benchmark is scored on one answer per question, so it samples greedily, as TalkToEBM's benchmark does.\nclient RouterBenchmarkBase = openai.OpenRouterClient.new(model = \"openrouter/free\", api_key = env.OPENROUTER_API_KEY, temperature = 0.0, max_tokens = 16000);\n\nclient RouterBenchmark = ai.clients.Retry.new(inner = RouterBenchmarkBase, max_attempts = 3);\n\nclient RouterLong = ai.clients.Retry.new(inner = RouterLongBase, max_attempts = 3);\n\nclass GraphPromptOptions {\n expert_description: string,\n graph_description: string,\n dataset_description: string,\n task_description: string,\n}\n\nfunction default_graph_options() -> GraphPromptOptions {\n GraphPromptOptions {\n expert_description: \"an expert statistician and data scientist\",\n graph_description: \"\",\n dataset_description: \"\",\n task_description: \"Describe the general pattern of the graph.\",\n }\n}\n\n// Prompt text helpers. Pure functions, so the Jinja `{% if %}` blocks become plain BAML.\n\nfunction graph_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You interpret global explanations produced by a Generalized Additive Model (GAM). Base every answer on the graph from the user, on any other information you are given, and on your knowledge of the real world. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\n// How to read the series of the classes. Only the graph of a multiclass classifier has \"Class: \" lines, so the prompt\n// of a binary classifier or a regressor gets no text about classes.\nfunction multiclass_format(graph: string) -> string {\n if (graph.includes(\"Class: \")) {\n return \"\\nA graph of a multiclass classifier repeats items 3 to 5 once for each class, after a line \\\"Class: <name>\\\". In every bin, the scores have the same mean over all the classes of the model, including classes that the graph leaves out. Only the differences between the classes change the predicted probabilities, so compare the classes with each other within each bin.\";\n };\n \"\"\n}\n\nfunction describe_graph_msg(graph: string, opts: GraphPromptOptions) -> string {\n let msg = `\n <graph_format>\n The graph is a JSON object. Its keys are the x-axis and its values are the y-axis. For a continuous feature, each key is an interval where the function predicts the same value. For a categorical feature, each key is a possible value of the feature.\n The graph contains these items:\n 1. The name of the feature.\n 2. The type of the feature (continuous, categorical, or boolean).\n 3. The mean values.\n 4. The lower bounds of the confidence interval (optional).\n 5. The upper bounds of the confidence interval (optional).${multiclass_format(graph)}\n </graph_format>\n\n <graph>\n ${graph}\n </graph>\n `\n + \"\\n\\n\";\n if (opts.graph_description.length() > 0) {\n msg = msg + `<graph_description>\\n${opts.graph_description}\\n</graph_description>\\n\\n`;\n };\n if (opts.dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${opts.dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg + `<task>\\n${opts.task_description}\\n</task>`\n}\n\nfunction summarize_ebm_system_msg(expert_description: string) -> string {\n `You are ${expert_description}. You provide an overall summary of a Generalized Additive Model (GAM). The model consists of different graphs, and each graph shows the effect of one input feature. Answer in plain prose paragraphs, because a person reads your answer.`\n}\n\nfunction summarize_ebm_msg(\n feature_importances: string,\n graph_descriptions: string,\n dataset_description: string,\n) -> string {\n let msg = `\n <feature_importances>\n ${feature_importances}\n </feature_importances>\n\n <graph_descriptions>\n ${graph_descriptions}\n </graph_descriptions>\n `\n + \"\\n\\n\";\n if (dataset_description.length() > 0) {\n msg = msg + `<dataset_description>\\n${dataset_description}\\n</dataset_description>\\n\\n`;\n };\n msg\n + `\n <instructions>\n Summarize the model. The feature importances are global, and there is exactly one graph description for each feature.\n Cover the most important features and their effect on the outcome. Leave out unimportant features and effects, so the summary stays short.\n Include any surprising pattern, because the reader wants to know where the model differs from expectation.\n </instructions>\n `\n}\n\n// Request text shared by the second turn and the replay of that turn in the third.\nfunction surprises_request() -> string {\n `\n <instructions>\n Study the graph carefully. Highlight any region that you find surprising or counterintuitive, and name its range or category so the reader can find it. Suggest why the behavior is surprising and what may have caused it.\n </instructions>\n `\n}\n\nfunction final_description_request(num_sentences: int) -> string {\n `\n <instructions>\n Describe the graph in at most ${num_sentences} sentences.\n Include the important surprising patterns, because the reader uses them to decide where to look closer.\n The reader knows that the graph is from a Generalized Additive Model (GAM), so leave that background out.\n </instructions>\n `\n}\n\n// describe_graph_cot, turn 1: system + describe_graph prompt.\nfunction GraphFirstPass(graph: string, opts: GraphPromptOptions) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n `\n}\n\n// Turn 2: surprising regions. `first_pass` is the assistant reply from turn 1.\nfunction GraphSurprises(graph: string, first_pass: string, opts: GraphPromptOptions) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n `\n}\n\n// Turn 3: brief final description.\nfunction GraphFinalDescription(\n graph: string,\n first_pass: string,\n surprises: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${describe_graph_msg(graph, opts)}\n ${role(\"assistant\")}\n ${first_pass}\n ${role(\"user\")}\n ${surprises_request()}\n ${role(\"assistant\")}\n ${surprises}\n ${role(\"user\")}\n ${final_description_request(num_sentences)}\n `\n}\n\n// summarize_ebm, turn 1.\nfunction EbmSummaryDraft(\n feature_importances: string,\n graph_descriptions: string,\n opts: GraphPromptOptions,\n) -> string {\n client: RouterLong\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description)}\n `\n}\n\n// summarize_ebm, turn 2: shorten the draft.\nfunction EbmSummaryShorten(\n feature_importances: string,\n graph_descriptions: string,\n draft: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n) -> string {\n client: Router\n prompt: `\n ${role(\"system\")}\n ${summarize_ebm_system_msg(opts.expert_description)}\n ${role(\"user\")}\n ${summarize_ebm_msg(feature_importances, graph_descriptions, opts.dataset_description)}\n ${role(\"assistant\")}\n ${draft}\n ${role(\"user\")}\n <instructions>\n Shorten your summary to at most ${num_sentences} sentences. Keep the most important information, because the reader wants the main effects and surprises.\n </instructions>\n `\n}\n\n// Token accounting.\n\nclass TokenUsage {\n input_tokens: int,\n output_tokens: int,\n}\n\n// An LLM result plus the tokens the provider reported for producing it.\nclass Counted {\n text: string,\n usage: TokenUsage,\n}\n\nclass TokenBudget {\n estimated_tokens: int,\n status: \"ok\" | \"warn\" | \"over\",\n}\n\nfunction add_usage(a: TokenUsage, b: TokenUsage) -> TokenUsage {\n TokenUsage {\n input_tokens: a.input_tokens + b.input_tokens,\n output_tokens: a.output_tokens + b.output_tokens,\n }\n}\n\n// Rough estimate (~3 chars/token): graph text is mostly digits and punctuation, which tokenize worse than prose.\n// No tokenizer is available before a call, so this is a guard rail; the provider's own error is the real limit.\nfunction estimate_tokens(text: string) -> int {\n (text.length() + 2) / 3\n}\n\n// \"ok\" within `max_tokens`; \"warn\" up to `max_tokens` plus `overbudget_percent`; \"over\" beyond that.\nfunction check_token_budget(text: string, max_tokens: int, overbudget_percent: int) -> TokenBudget {\n let estimated = estimate_tokens(text);\n let hard_limit = max_tokens * (100 + overbudget_percent) / 100;\n if (estimated > hard_limit) {\n return TokenBudget { estimated_tokens: estimated, status: \"over\" };\n };\n if (estimated > max_tokens) {\n return TokenBudget { estimated_tokens: estimated, status: \"warn\" };\n };\n TokenBudget { estimated_tokens: estimated, status: \"ok\" }\n}\n\n// Runs one text-producing LLM function and keeps the provider's token usage. A count that the provider did not report\n// is zero.\nfunction run_counted(spec: ai.FunctionSpec<string>, client: ai.Client? = null) -> Counted {\n let result = ai.Agent.new(client = client).run(spec);\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0 };\n for (let entry in result.usage) {\n usage\n = add_usage(\n usage,\n TokenUsage {\n input_tokens: entry.usage.input_tokens ?? 0,\n output_tokens: entry.usage.output_tokens ?? 0,\n },\n );\n }\n Counted { text: result.value, usage: usage }\n}\n\n// Entry points called from Python.\n\nfunction describe_graph_cot(\n graph: string,\n num_sentences: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n //# Three sequential assistant turns, each seeing the previous replies\n let long = select_client(client, model, long_defaults());\n let short = select_client(client, model, short_defaults());\n let first_pass = run_counted(GraphFirstPass@spec(graph, opts), client = long);\n let surprises = run_counted(GraphSurprises@spec(graph, first_pass.text, opts), client = short);\n let final = run_counted(\n GraphFinalDescription@spec(graph, first_pass.text, surprises.text, num_sentences, opts),\n client = short,\n );\n Counted {\n text: final.text,\n usage: add_usage(add_usage(first_pass.usage, surprises.usage), final.usage),\n }\n}\n\n// `num_sentences` null keeps the draft summary as is.\nfunction summarize_ebm(\n feature_importances: string,\n graph_descriptions: string,\n num_sentences: int?,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let draft = run_counted(\n EbmSummaryDraft@spec(feature_importances, graph_descriptions, opts),\n client = select_client(client, model, long_defaults()),\n );\n if (num_sentences == null) {\n return draft;\n };\n let shortened = run_counted(\n EbmSummaryShorten@spec(feature_importances, graph_descriptions, draft.text, num_sentences, opts),\n client = select_client(client, model, short_defaults()),\n );\n Counted { text: shortened.text, usage: add_usage(draft.usage, shortened.usage) }\n}\n\n// Full EBM description: one chain per feature graph (concurrently), then the summary.\n// `graphs[i]` is the text of the graph of `feature_names[i]`.\nfunction describe_ebm(\n feature_names: string[],\n graphs: string[],\n feature_importances: string,\n num_sentences: int,\n graph_sentences: int,\n max_concurrency: int,\n opts: GraphPromptOptions,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n if (feature_names.length() != graphs.length()) {\n throw baml.errors.InvalidArgument { message: \"feature_names and graphs must have the same length\" };\n };\n //# One description chain per graph, at most `max_concurrency` at a time\n let limit = baml.spawn.Limit.new(max_concurrency);\n let futures = graphs.map((graph) -> {\n spawn with limit {\n describe_graph_cot(graph, graph_sentences, opts, client = client, model = model)\n }\n });\n let described: Counted[] = await baml.future.all(futures);\n\n let usage = TokenUsage { input_tokens: 0, output_tokens: 0 };\n let lines: string[] = [];\n let i = 0;\n while (i < described.length()) {\n let item = described[i];\n usage = add_usage(usage, item.usage);\n lines.push(`${feature_names[i]}: ${item.text}`);\n i = i + 1;\n }\n //# Summarize all the per-graph descriptions\n let summary = summarize_ebm(feature_importances, lines.join(\"\\n\\n\"), num_sentences, opts, client = client, model = model);\n Counted { text: summary.text, usage: add_usage(usage, summary.usage) }\n}\n\n// Benchmark (benchmark/): the questions end with \"Think step by step.\", so turn 1 reasons freely and turn 2\n// extracts the final answer in the format the scorer parses.\n\nfunction BenchmarkReasoning(question: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n `\n}\n\nfunction BenchmarkAnswer(question: string, reasoning: string, answer_format: string) -> string {\n client: RouterBenchmark\n prompt: `\n ${role(\"system\")}\n ${graph_system_msg(\"an expert statistician and data scientist\")}\n ${role(\"user\")}\n ${question}\n ${role(\"assistant\")}\n ${reasoning}\n ${role(\"user\")}\n <instructions>\n Summarize your response by answering with ${answer_format}. Reply with the answer only, because a program parses your reply.\n </instructions>\n `\n}\n\nfunction answer_benchmark_question(\n question: string,\n answer_format: string,\n client: ai.Client? = null,\n model: ModelSpec? = null,\n) -> Counted {\n let selected = select_client(client, model, benchmark_defaults());\n let reasoning = run_counted(BenchmarkReasoning@spec(question), client = selected);\n let answer = run_counted(BenchmarkAnswer@spec(question, reasoning.text, answer_format), client = selected);\n Counted { text: answer.text, usage: add_usage(reasoning.usage, answer.usage) }\n}\n",
|
|
31
31
|
"talk_to_ebm_test.baml": "// Offline tests for talk_to_ebm.baml: pure helpers and the pipelines on a scripted client.\n\n// A model turn that holds `text` and reports `input_tokens` and `output_tokens`. The runner reads the usage of a turn\n// from its calls.\nfunction scripted_turn(text: string, input_tokens: int, output_tokens: int) -> ai.ModelTurn {\n ai.ModelTurn {\n parsed_output: null,\n content: [ai.content.Text { text: text }],\n stop_reason: ai.content.StopReason.Complete,\n resolved_model: null,\n requested_model: null,\n calls: [\n ai.events.ModelCall {\n resolved_model: ai.events.ModelKey { provider: \"scripted\", model: null },\n usage: ai.events.Usage {\n input_tokens: input_tokens,\n output_tokens: output_tokens,\n cached_input_tokens: null,\n reasoning_tokens: null,\n cache_write_input_tokens: null,\n uncached_input_tokens: null,\n },\n timing: ai.events.Timing { start_time: null, duration: null, ttft: null },\n http_request: null,\n http_response: null,\n sse_responses: null,\n },\n ],\n child_turns: [],\n error: null,\n }\n}\n\n// Answers every call with \"R\" and reports 10 input / 5 output tokens; counts the calls.\nclass ScriptedClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"scripted\"\n }\n\n function requested_model(self) -> string {\n \"scripted\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://scripted.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n scripted_turn(\"R\", 10, 5)\n }\n }\n}\n\ntest \"estimate_tokens rounds up at 3 chars per token\" {\n assert.equal(estimate_tokens(\"\"), 0);\n assert.equal(estimate_tokens(\"abc\"), 1);\n assert.equal(estimate_tokens(\"abcd\"), 2)\n}\n\ntest \"check_token_budget: ok, warn, over\" {\n // 300 chars -> 100 tokens estimated\n let text = \"x\".repeat(300);\n assert.equal(check_token_budget(text, 100, 50).status, \"ok\");\n assert.equal(check_token_budget(text, 80, 50).status, \"warn\");\n assert.equal(check_token_budget(text, 60, 50).status, \"over\");\n assert.equal(check_token_budget(text, 80, 50).estimated_tokens, 100)\n}\n\ntest \"add_usage sums both counters\" {\n let total = add_usage(\n TokenUsage { input_tokens: 1, output_tokens: 2 },\n TokenUsage { input_tokens: 10, output_tokens: 20 },\n );\n assert.equal(total.input_tokens, 11);\n assert.equal(total.output_tokens, 22)\n}\n\ntest \"describe_graph_msg includes optional sections only when set\" {\n let bare = describe_graph_msg(\"GRAPH\", default_graph_options());\n assert.is_true(bare.includes(\"GRAPH\"));\n assert.is_true(bare.includes(\"Describe the general pattern of the graph.\"));\n assert.is_true(!bare.includes(\"<dataset_description>\"));\n\n let opts = GraphPromptOptions { ...default_graph_options(), dataset_description: \"A pneumonia dataset\" };\n assert.is_true(describe_graph_msg(\"GRAPH\", opts).includes(\"A pneumonia dataset\"))\n}\n\ntest \"describe_graph_msg says how to read the classes of a multiclass graph\" {\n let msg = describe_graph_msg(\"Feature Name: a\\nClass: high\\nMeans: {}\", default_graph_options());\n assert.is_true(\n msg.includes(\"(optional).\\nA graph of a multiclass classifier repeats items 3 to 5\"),\n );\n assert.is_true(msg.includes(\"including classes that the graph leaves out\"));\n assert.is_true(msg.includes(\"compare the classes with each other within each bin.\"));\n assert.is_true(position_of(msg, \"Class: <name>\") < position_of(msg, \"</graph_format>\"))\n}\n\ntest \"describe_graph_msg of a graph with one output says nothing about classes\" {\n let msg = describe_graph_msg(\"Feature Name: a\\nMeans: {}\", default_graph_options());\n assert.is_true(!msg.includes(\"multiclass\"));\n assert.is_true(\n msg.includes(\"5. The upper bounds of the confidence interval (optional).\\n</graph_format>\"),\n )\n}\n\ntest \"describe_graph_cot makes three calls and sums usage\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = scripted);\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 3);\n assert.equal(result.usage.input_tokens, 30);\n assert.equal(result.usage.output_tokens, 15)\n}\n\ntest \"summarize_ebm shortens only when num_sentences is set\" {\n let draft_only = ScriptedClient { calls: 0 };\n summarize_ebm(\"IMP\", \"DESC\", null, default_graph_options(), client = draft_only);\n assert.equal(draft_only.calls, 1);\n\n let shortened = ScriptedClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = shortened);\n assert.equal(shortened.calls, 2);\n assert.equal(result.usage.input_tokens, 20)\n}\n\ntest \"describe_ebm: one chain per graph plus the summary\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_ebm([\"age\", \"bmi\"], [\"G1\", \"G2\"], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted);\n // 2 graphs x 3 turns + draft + shorten. The two chains run at the same time, so `self.calls += 1` of the\n // scripted client can lose an increment; each call reports 10 input and 5 output tokens, so the usage counts the\n // calls without a shared counter.\n assert.equal(result.usage.input_tokens, 8 * 10);\n assert.equal(result.usage.output_tokens, 8 * 5)\n}\n\ntest \"describe_ebm rejects mismatched names and graphs\" {\n let scripted = ScriptedClient { calls: 0 };\n let outcome = describe_ebm([\"age\"], [], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted) catch (e) {\n baml.errors.InvalidArgument => null,\n };\n assert.is_true(outcome == null);\n assert.equal(scripted.calls, 0)\n}\n\ntest \"answer_benchmark_question reasons, then extracts the answer\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = scripted);\n assert.equal(result.text, \"R\");\n assert.equal(scripted.calls, 2);\n assert.equal(result.usage.input_tokens, 20);\n assert.equal(result.usage.output_tokens, 10)\n}\n\ntest \"local_user_msg prepends the global explanation and follows the audience\" {\n let layperson = local_user_msg(\"GLOBAL\", \"PRED\", \"QUESTION\", \"FACTS\", \"layperson\", \"\");\n assert.is_true(layperson.includes(\"GLOBAL\"));\n assert.is_true(layperson.includes(\"FACTS\"));\n assert.is_true(layperson.includes(\"model numbers\"));\n assert.is_true(!layperson.includes(\"<dataset_description>\"));\n\n let expert = local_user_msg(\"GLOBAL\", \"PRED\", \"QUESTION\", \"FACTS\", \"data_scientist\", \"A bank dataset\");\n assert.is_true(expert.includes(\"confidence intervals\"));\n assert.is_true(expert.includes(\"A bank dataset\"))\n}\n\ntest \"explain_local makes three calls and sums usage\" {\n let scripted = ScriptedClient { calls: 0 };\n let facts = LocalFactTexts {\n prediction: \"P\",\n counterfactual: \"C\",\n sufficient_reason: \"S\",\n completions: \"K\",\n };\n let result = explain_local(\"GLOBAL\", facts, \"layperson\", 5, default_graph_options(), client = scripted);\n assert.equal(scripted.calls, 3);\n assert.equal(result.counterfactual, \"R\");\n assert.equal(result.sufficient_reason, \"R\");\n assert.equal(result.completions, \"R\");\n assert.equal(result.usage.input_tokens, 30);\n assert.equal(result.usage.output_tokens, 15)\n}\n\n// Answers every call with its own rendered prompt, so a test can see what each call was given.\nclass EchoClient {\n implements ai.Client {\n function provider(self) -> string {\n \"echo\"\n }\n\n function requested_model(self) -> string {\n \"echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n let prompt = input.prompt(ai.internal.build_output_format(input.output_type));\n scripted_turn(prompt.text(), 1, 1)\n }\n }\n}\n\n// Index of `needle` in `text`, or -1 when absent, so positions can be ordered.\nfunction position_of(text: string, needle: string) -> int {\n let found = text.index_of(needle);\n if (found == null) {\n return -1;\n };\n found\n}\n\ntest \"local_system_msg names the expert and forbids invented facts\" {\n let msg = local_system_msg(\"a credit risk analyst\");\n assert.is_true(msg.includes(\"You are a credit risk analyst.\"));\n assert.is_true(msg.includes(\"invented number\"))\n}\n\ntest \"audience_guidance differs per audience\" {\n let layperson = audience_guidance(\"layperson\");\n let expert = audience_guidance(\"data_scientist\");\n assert.is_true(layperson.includes(\"not technical\"));\n assert.is_true(!layperson.includes(\"data scientist\"));\n assert.is_true(expert.includes(\"data scientist\"));\n assert.is_true(!expert.includes(\"not technical\"))\n}\n\ntest \"local_user_msg orders the global explanation, dataset, prediction, facts, guidance and question\" {\n let msg = local_user_msg(\"GLOBAL\", \"PRED\", \"QUESTION\", \"FACTS\", \"layperson\", \"DATASET\");\n let positions = [\n position_of(msg, \"GLOBAL\"),\n position_of(msg, \"DATASET\"),\n position_of(msg, \"PRED\"),\n position_of(msg, \"FACTS\"),\n position_of(msg, \"not technical\"),\n position_of(msg, \"QUESTION\"),\n ];\n assert.is_true(positions[0] >= 0);\n assert.is_true(positions[0] < positions[1]);\n assert.is_true(positions[1] < positions[2]);\n assert.is_true(positions[2] < positions[3]);\n assert.is_true(positions[3] < positions[4]);\n assert.is_true(positions[4] < positions[5])\n}\n\ntest \"question functions state the sentence limit and their own query\" {\n assert.is_true(counterfactual_question(3).includes(\"at most 3 sentences\"));\n assert.is_true(counterfactual_question(3).includes(\"minimum change required\"));\n assert.is_true(sufficient_reason_question(4).includes(\"at most 4 sentences\"));\n assert.is_true(sufficient_reason_question(4).includes(\"sufficient reason\"));\n assert.is_true(completions_question(6).includes(\"at most 6 sentences\"));\n assert.is_true(completions_question(6).includes(\"count completions\"))\n}\n\ntest \"each local function gets its own facts, question and system role\" {\n let facts = LocalFactTexts {\n prediction: \"PRED-FACT\",\n counterfactual: \"CF-FACT\",\n sufficient_reason: \"SR-FACT\",\n completions: \"CO-FACT\",\n };\n let opts = GraphPromptOptions { ...default_graph_options(), expert_description: \"an EXPERT\" };\n\n let cf = LocalCounterfactual@spec(\"GLOBAL\", facts, \"layperson\", 3, opts).prompt().text();\n let sr = LocalSufficientReason@spec(\"GLOBAL\", facts, \"layperson\", 4, opts).prompt().text();\n let co = LocalCompletions@spec(\"GLOBAL\", facts, \"layperson\", 6, opts).prompt().text();\n\n for (let prompt in [cf, sr, co]) {\n assert.is_true(prompt.includes(\"You are an EXPERT.\"));\n assert.is_true(prompt.includes(\"GLOBAL\"));\n assert.is_true(prompt.includes(\"PRED-FACT\"))\n }\n ;\n\n assert.is_true(\n cf.includes(\"CF-FACT\")\n && cf.includes(\"at most 3 sentences\")\n && cf.includes(\"minimum change required\"),\n );\n assert.is_true(!cf.includes(\"SR-FACT\") && !cf.includes(\"CO-FACT\"));\n assert.is_true(\n sr.includes(\"SR-FACT\")\n && sr.includes(\"at most 4 sentences\")\n && sr.includes(\"sufficient reason\"),\n );\n assert.is_true(!sr.includes(\"CF-FACT\") && !sr.includes(\"CO-FACT\"));\n assert.is_true(\n co.includes(\"CO-FACT\")\n && co.includes(\"at most 6 sentences\")\n && co.includes(\"count completions\"),\n );\n assert.is_true(!co.includes(\"CF-FACT\") && !co.includes(\"SR-FACT\"))\n}\n\ntest \"explain_local routes each answer to its own field\" {\n let facts = LocalFactTexts {\n prediction: \"PRED-FACT\",\n counterfactual: \"CF-FACT\",\n sufficient_reason: \"SR-FACT\",\n completions: \"CO-FACT\",\n };\n let result = explain_local(\"GLOBAL\", facts, \"data_scientist\", 5, default_graph_options(), client = EchoClient {});\n assert.is_true(result.counterfactual.includes(\"CF-FACT\"));\n assert.is_true(\n !result.counterfactual.includes(\"SR-FACT\") && !result.counterfactual.includes(\"CO-FACT\"),\n );\n assert.is_true(result.sufficient_reason.includes(\"SR-FACT\"));\n assert.is_true(\n !result.sufficient_reason.includes(\"CF-FACT\")\n && !result.sufficient_reason.includes(\"CO-FACT\"),\n );\n assert.is_true(result.completions.includes(\"CO-FACT\"));\n assert.is_true(\n !result.completions.includes(\"CF-FACT\") && !result.completions.includes(\"SR-FACT\"),\n );\n assert.is_true(result.completions.includes(\"data scientist\"));\n assert.equal(result.usage.input_tokens, 3);\n assert.equal(result.usage.output_tokens, 3)\n}\n\n// Answers call n with \"REPLY-n\" and counts the calls, so a test can see which reply reached which later prompt.\n// Use it only for sequential pipelines: concurrent calls would race on the counter.\nclass SequenceClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"sequence\"\n }\n\n function requested_model(self) -> string {\n \"sequence\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://sequence.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n scripted_turn(`REPLY-${self.calls}`, 1, 1)\n }\n }\n}\n\n// The roles of a rendered prompt, joined, e.g. \"system,user,assistant,user\".\nfunction roles_of(prompt: ai.Prompt) -> string {\n prompt.messages()\n .map((m) -> {\n m.role\n })\n .join(\",\")\n}\n\n// The content of message `index` of a rendered prompt.\nfunction message_at(prompt: ai.Prompt, index: int) -> string {\n prompt.messages()[index].content\n}\n\ntest \"default_graph_options is a plain statistician with no extra context\" {\n let opts = default_graph_options();\n assert.equal(opts.expert_description, \"an expert statistician and data scientist\");\n assert.equal(opts.graph_description, \"\");\n assert.equal(opts.dataset_description, \"\");\n assert.equal(opts.task_description, \"Describe the general pattern of the graph.\")\n}\n\ntest \"estimate_tokens rounds up at every remainder\" {\n assert.equal(estimate_tokens(\"a\"), 1);\n assert.equal(estimate_tokens(\"ab\"), 1);\n assert.equal(estimate_tokens(\"abc\"), 1);\n assert.equal(estimate_tokens(\"abcd\"), 2);\n assert.equal(estimate_tokens(\"abcdef\"), 2);\n assert.equal(estimate_tokens(\"abcdefg\"), 3)\n}\n\ntest \"check_token_budget changes status exactly past the limits\" {\n // 300 chars -> 100 tokens. The hard limit is max_tokens * (100 + percent) / 100, rounded down.\n let text = \"x\".repeat(300);\n assert.equal(check_token_budget(text, 100, 0).status, \"ok\");\n assert.equal(check_token_budget(text, 99, 2).status, \"warn\");\n assert.equal(check_token_budget(text, 99, 1).status, \"over\");\n assert.equal(check_token_budget(text, 99, 0).status, \"over\");\n assert.equal(check_token_budget(text, 67, 50).status, \"warn\");\n assert.equal(check_token_budget(text, 66, 50).status, \"over\");\n assert.equal(check_token_budget(text, 66, 50).estimated_tokens, 100)\n}\n\ntest \"add_usage keeps the operands independent\" {\n let zero = TokenUsage { input_tokens: 0, output_tokens: 0 };\n let usage = TokenUsage { input_tokens: 3, output_tokens: 4 };\n assert.equal(add_usage(zero, usage).input_tokens, 3);\n assert.equal(add_usage(zero, usage).output_tokens, 4);\n assert.equal(add_usage(usage, usage).input_tokens, 6);\n assert.equal(add_usage(usage, usage).output_tokens, 8)\n}\n\ntest \"graph_system_msg names the expert and the GAM\" {\n let msg = graph_system_msg(\"a medical statistician\");\n assert.is_true(msg.includes(\"You are a medical statistician.\"));\n assert.is_true(msg.includes(\"Generalized Additive Model\"))\n}\n\ntest \"describe_graph_msg orders graph, graph description, dataset and task\" {\n let opts = GraphPromptOptions {\n expert_description: \"E\",\n graph_description: \"GRAPHDESC\",\n dataset_description: \"DATASET\",\n task_description: \"TASK\",\n };\n let msg = describe_graph_msg(\"GRAPH\", opts);\n assert.is_true(position_of(msg, \"GRAPH\") >= 0);\n assert.is_true(position_of(msg, \"GRAPH\") < position_of(msg, \"GRAPHDESC\"));\n assert.is_true(position_of(msg, \"GRAPHDESC\") < position_of(msg, \"DATASET\"));\n assert.is_true(position_of(msg, \"DATASET\") < position_of(msg, \"TASK\"));\n assert.is_true(msg.includes(\"<dataset_description>\"))\n}\n\ntest \"describe_graph_msg leaves out each optional section on its own\" {\n let only_graph = GraphPromptOptions { ...default_graph_options(), graph_description: \"GRAPHDESC\" };\n let msg = describe_graph_msg(\"GRAPH\", only_graph);\n assert.is_true(msg.includes(\"GRAPHDESC\"));\n assert.is_true(!msg.includes(\"<dataset_description>\"));\n\n let only_dataset = GraphPromptOptions { ...default_graph_options(), dataset_description: \"DATASET\" };\n let other = describe_graph_msg(\"GRAPH\", only_dataset);\n assert.is_true(other.includes(\"DATASET\"));\n assert.is_true(!other.includes(\"GRAPHDESC\"));\n\n let custom = GraphPromptOptions { ...default_graph_options(), task_description: \"MY TASK\" };\n let task = describe_graph_msg(\"GRAPH\", custom);\n assert.is_true(task.includes(\"MY TASK\"));\n assert.is_true(!task.includes(\"Describe the general pattern\"))\n}\n\ntest \"summarize_ebm_system_msg names the expert\" {\n let msg = summarize_ebm_system_msg(\"a risk analyst\");\n assert.is_true(msg.includes(\"You are a risk analyst.\"));\n assert.is_true(msg.includes(\"overall summary\"))\n}\n\ntest \"summarize_ebm_msg orders importances, descriptions and dataset, and the dataset is optional\" {\n let msg = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"DATASET\");\n assert.is_true(position_of(msg, \"IMPORTANCES\") >= 0);\n assert.is_true(position_of(msg, \"IMPORTANCES\") < position_of(msg, \"DESCRIPTIONS\"));\n assert.is_true(position_of(msg, \"DESCRIPTIONS\") < position_of(msg, \"DATASET\"));\n assert.is_true(position_of(msg, \"DATASET\") < position_of(msg, \"<instructions>\"));\n\n let bare = summarize_ebm_msg(\"IMPORTANCES\", \"DESCRIPTIONS\", \"\");\n assert.is_true(!bare.includes(\"<dataset_description>\"));\n assert.is_true(bare.includes(\"<instructions>\"))\n}\n\ntest \"describe_graph_cot prompts: roles, content and sentence limit per turn\" {\n let opts = GraphPromptOptions { ...default_graph_options(), expert_description: \"an EXPERT\" };\n\n let first = GraphFirstPass@spec(\"GRAPH\", opts).prompt();\n assert.equal(roles_of(first), \"system,user\");\n assert.is_true(message_at(first, 0).includes(\"You are an EXPERT.\"));\n assert.is_true(message_at(first, 1).includes(\"GRAPH\"));\n\n let second = GraphSurprises@spec(\"GRAPH\", \"FIRST\", opts).prompt();\n assert.equal(roles_of(second), \"system,user,assistant,user\");\n assert.is_true(message_at(second, 1).includes(\"GRAPH\"));\n assert.equal(message_at(second, 2).trim(), \"FIRST\");\n assert.is_true(message_at(second, 3).includes(\"surprising or counterintuitive\"));\n\n let third = GraphFinalDescription@spec(\"GRAPH\", \"FIRST\", \"SURPRISES\", 7, opts).prompt();\n assert.equal(roles_of(third), \"system,user,assistant,user,assistant,user\");\n assert.equal(message_at(third, 2).trim(), \"FIRST\");\n assert.equal(message_at(third, 4).trim(), \"SURPRISES\");\n assert.is_true(message_at(third, 5).includes(\"at most 7 sentence\"))\n}\n\ntest \"describe_graph_cot feeds each reply to the next turn in order\" {\n let sequence = SequenceClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = sequence);\n assert.equal(sequence.calls, 3);\n assert.equal(result.text, \"REPLY-3\");\n assert.equal(result.usage.input_tokens, 3);\n assert.equal(result.usage.output_tokens, 3);\n\n // The echo client returns the last prompt, which must hold both earlier replies in order.\n let echo = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = EchoClient {});\n assert.is_true(echo.text.includes(\"GRAPH\"));\n assert.is_true(echo.text.includes(\"at most 7 sentence\"));\n assert.is_true(position_of(echo.text, \"Highlight any region\") >= 0);\n assert.is_true(\n position_of(echo.text, \"Describe the graph in at most\")\n > position_of(echo.text, \"Highlight any region\"),\n )\n}\n\ntest \"describe_graph_cot passes the second and third turns the replies of the earlier turns\" {\n let sequence = SequenceClient { calls: 0 };\n let first = GraphFirstPass@spec(\"GRAPH\", default_graph_options());\n let reply_one = run_counted(first, client = sequence);\n assert.equal(reply_one.text, \"REPLY-1\");\n let reply_two = run_counted(GraphSurprises@spec(\"GRAPH\", reply_one.text, default_graph_options()), client = sequence);\n assert.equal(reply_two.text, \"REPLY-2\");\n let third = GraphFinalDescription@spec(\"GRAPH\", reply_one.text, reply_two.text, 5, default_graph_options())\n .prompt();\n assert.equal(message_at(third, 2).trim(), \"REPLY-1\");\n assert.equal(message_at(third, 4).trim(), \"REPLY-2\")\n}\n\ntest \"summarize_ebm prompts: roles, content and sentence limit\" {\n let opts = GraphPromptOptions {\n ...default_graph_options(),\n expert_description: \"an EXPERT\",\n dataset_description: \"DATASET\",\n };\n let draft = EbmSummaryDraft@spec(\"IMPORTANCES\", \"DESCRIPTIONS\", opts).prompt();\n assert.equal(roles_of(draft), \"system,user\");\n assert.is_true(message_at(draft, 0).includes(\"You are an EXPERT.\"));\n assert.is_true(message_at(draft, 1).includes(\"IMPORTANCES\"));\n assert.is_true(message_at(draft, 1).includes(\"DESCRIPTIONS\"));\n assert.is_true(message_at(draft, 1).includes(\"DATASET\"));\n\n let shorten = EbmSummaryShorten@spec(\"IMPORTANCES\", \"DESCRIPTIONS\", \"DRAFT\", 12, opts).prompt();\n assert.equal(roles_of(shorten), \"system,user,assistant,user\");\n assert.is_true(message_at(shorten, 1).includes(\"IMPORTANCES\"));\n assert.equal(message_at(shorten, 2).trim(), \"DRAFT\");\n assert.is_true(message_at(shorten, 3).includes(\"at most 12 sentences\"))\n}\n\ntest \"summarize_ebm returns the draft when num_sentences is null, else the shortened reply\" {\n let draft_only = SequenceClient { calls: 0 };\n let kept = summarize_ebm(\"IMP\", \"DESC\", null, default_graph_options(), client = draft_only);\n assert.equal(kept.text, \"REPLY-1\");\n assert.equal(kept.usage.input_tokens, 1);\n assert.equal(kept.usage.output_tokens, 1);\n\n let shortened = SequenceClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = shortened);\n assert.equal(result.text, \"REPLY-2\");\n assert.equal(result.usage.input_tokens, 2);\n assert.equal(result.usage.output_tokens, 2);\n\n // The shorten prompt holds the draft and the limit.\n let echo = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = EchoClient {});\n assert.is_true(echo.text.includes(\"at most 30 sentences\"));\n assert.is_true(echo.text.includes(\"IMP\"));\n assert.is_true(echo.text.includes(\"DESC\"))\n}\n\ntest \"describe_ebm pairs each feature name with its own description, in order\" {\n let result = describe_ebm(\n [\"age\", \"bmi\"],\n [\"G-AGE\", \"G-BMI\"],\n \"IMP\",\n 30,\n 7,\n 2,\n default_graph_options(),\n client = EchoClient {},\n );\n // The summary draft echoes the \"name: description\" lines; each description echoes its own graph.\n assert.is_true(position_of(result.text, \"age: \") >= 0);\n assert.is_true(position_of(result.text, \"age: \") < position_of(result.text, \"bmi: \"));\n assert.is_true(position_of(result.text, \"G-AGE\") < position_of(result.text, \"bmi: \"));\n assert.is_true(position_of(result.text, \"bmi: \") < position_of(result.text, \"G-BMI\"));\n assert.is_true(result.text.includes(\"at most 30 sentences\"));\n assert.is_true(result.text.includes(\"at most 7 sentence\"));\n assert.is_true(result.text.includes(\"IMP\"))\n}\n\ntest \"describe_ebm gives the same usage whatever the concurrency\" {\n let one = describe_ebm(\n [\"a\", \"b\", \"c\"],\n [\"G1\", \"G2\", \"G3\"],\n \"IMP\",\n 30,\n 7,\n 1,\n default_graph_options(),\n client = EchoClient {},\n );\n let many = describe_ebm(\n [\"a\", \"b\", \"c\"],\n [\"G1\", \"G2\", \"G3\"],\n \"IMP\",\n 30,\n 7,\n 8,\n default_graph_options(),\n client = EchoClient {},\n );\n // 3 graphs x 3 turns + draft + shorten = 11 calls at 1 input and 1 output token each.\n assert.equal(one.usage.input_tokens, 11);\n assert.equal(one.usage.output_tokens, 11);\n assert.equal(many.usage.input_tokens, 11);\n assert.equal(many.usage.output_tokens, 11)\n}\n\ntest \"describe_ebm with no graphs only summarizes\" {\n let scripted = ScriptedClient { calls: 0 };\n let result = describe_ebm([], [], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted);\n assert.equal(scripted.calls, 2);\n assert.equal(result.usage.input_tokens, 20)\n}\n\ntest \"describe_ebm rejects more names than graphs and more graphs than names\" {\n let scripted = ScriptedClient { calls: 0 };\n let more_graphs = describe_ebm([], [\"G\"], \"IMP\", 30, 7, 2, default_graph_options(), client = scripted) catch (e) {\n baml.errors.InvalidArgument => null,\n };\n assert.is_true(more_graphs == null);\n assert.equal(scripted.calls, 0)\n}\n\ntest \"benchmark prompts: roles, question, reasoning and answer format\" {\n let reasoning = BenchmarkReasoning@spec(\"QUESTION\").prompt();\n assert.equal(roles_of(reasoning), \"system,user\");\n assert.is_true(\n message_at(reasoning, 0).includes(\"You are an expert statistician and data scientist.\"),\n );\n assert.is_true(message_at(reasoning, 1).includes(\"QUESTION\"));\n\n let answer = BenchmarkAnswer@spec(\"QUESTION\", \"REASONING\", \"'a' or 'b'\").prompt();\n assert.equal(roles_of(answer), \"system,user,assistant,user\");\n assert.is_true(message_at(answer, 1).includes(\"QUESTION\"));\n assert.equal(message_at(answer, 2).trim(), \"REASONING\");\n assert.is_true(message_at(answer, 3).includes(\"'a' or 'b'\"));\n assert.is_true(message_at(answer, 3).includes(\"Reply with the answer only\"))\n}\n\ntest \"answer_benchmark_question hands the reasoning to the answer turn\" {\n let sequence = SequenceClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = sequence);\n assert.equal(sequence.calls, 2);\n assert.equal(result.text, \"REPLY-2\");\n assert.equal(result.usage.input_tokens, 2);\n assert.equal(result.usage.output_tokens, 2);\n\n let echo = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = EchoClient {});\n assert.is_true(echo.text.includes(\"'a' or 'b'\"));\n assert.is_true(echo.text.includes(\"QUESTION\"));\n assert.is_true(echo.text.includes(\"Summarize your response\"))\n}\n\n// A model turn that holds `text` and reports 1 input and 1 output token.\nfunction text_turn(text: string) -> ai.ModelTurn {\n scripted_turn(text, 1, 1)\n}\n\n// Answers call n with \"Rn<<\" + its rendered prompt + \">>\". A later prompt then shows which earlier reply sits where.\n// Use it only for sequential pipelines: concurrent calls would race on the counter.\nclass TaggedEchoClient {\n calls: int,\n\n implements ai.Client {\n function provider(self) -> string {\n \"tagged-echo\"\n }\n\n function requested_model(self) -> string {\n \"tagged-echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://tagged-echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n self.calls += 1;\n let prompt = input.prompt(ai.internal.build_output_format(input.output_type));\n text_turn(`R${self.calls}<<${prompt.text()}>>`)\n }\n }\n}\n\n// Answers \"R\" to graph prompts and echoes the summary prompts, so the graph descriptions stay predictable.\nclass SummaryEchoClient {\n implements ai.Client {\n function provider(self) -> string {\n \"summary-echo\"\n }\n\n function requested_model(self) -> string {\n \"summary-echo\"\n }\n\n function render(self, input: ai.ModelTurnInput) -> baml.http.Request {\n baml.http.Request.from_url(\"http://summary-echo.invalid\")\n }\n\n function invoke(self, input: ai.ModelTurnInput) -> ai.ModelTurn {\n let text = input.prompt(ai.internal.build_output_format(input.output_type)).text();\n if (text.includes(\"overall summary of a Generalized Additive Model\")) {\n return text_turn(text);\n };\n text_turn(\"R\")\n }\n }\n}\n\ntest \"describe_graph_cot puts the first reply in the second and third turns and the second reply in the third\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = describe_graph_cot(\"GRAPH\", 7, default_graph_options(), client = tagged);\n assert.is_true(result.text.starts_with(\"R3<<\"));\n // The third prompt holds R1 (with the first prompt inside it) before R2, and R2 holds R1 too.\n let r1 = position_of(result.text, \"R1<<\");\n let r2 = position_of(result.text, \"R2<<\");\n assert.is_true(r1 > 0);\n assert.is_true(r1 < r2);\n assert.is_true(position_of(result.text.slice(r2, result.text.length()), \"R1<<\") > 0)\n}\n\ntest \"describe_ebm joins the per-feature descriptions with a blank line\" {\n let result = describe_ebm(\n [\"age\", \"bmi\"],\n [\"G1\", \"G2\"],\n \"IMP\",\n 30,\n 7,\n 2,\n default_graph_options(),\n client = SummaryEchoClient {},\n );\n assert.is_true(result.text.includes(\"age: R\\n\\nbmi: R\"))\n}\n\ntest \"answer_benchmark_question puts the reasoning reply in the answer prompt\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = answer_benchmark_question(\"QUESTION\", \"'a' or 'b'\", client = tagged);\n assert.is_true(result.text.starts_with(\"R2<<\"));\n assert.is_true(result.text.includes(\"R1<<\"));\n assert.is_true(\n position_of(result.text, \"R1<<\") < position_of(result.text, \"Summarize your response\"),\n )\n}\n\ntest \"summarize_ebm puts the draft reply in the shorten prompt\" {\n let tagged = TaggedEchoClient { calls: 0 };\n let result = summarize_ebm(\"IMP\", \"DESC\", 30, default_graph_options(), client = tagged);\n assert.is_true(result.text.starts_with(\"R2<<\"));\n assert.is_true(position_of(result.text, \"R1<<\") > 0);\n assert.is_true(\n position_of(result.text, \"R1<<\") < position_of(result.text, \"at most 30 sentences\"),\n )\n}\n",
|
|
32
32
|
}
|