evalmarket-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalmarket/__init__.py +1 -0
- evalmarket/__main__.py +5 -0
- evalmarket/_version.py +3 -0
- evalmarket/braintrust/__init__.py +1 -0
- evalmarket/braintrust/client.py +153 -0
- evalmarket/braintrust/experiments.py +67 -0
- evalmarket/braintrust/importer.py +309 -0
- evalmarket/braintrust/models.py +169 -0
- evalmarket/braintrust/resources.py +21 -0
- evalmarket/braintrust/validation.py +368 -0
- evalmarket/cli.py +103 -0
- evalmarket/entrypoint.py +41 -0
- evalmarket/json_data.py +71 -0
- evalmarket/langsmith/__init__.py +1 -0
- evalmarket/langsmith/importer.py +246 -0
- evalmarket/langsmith/models.py +52 -0
- evalmarket/langsmith/playground.py +123 -0
- evalmarket/langsmith/prompts.py +72 -0
- evalmarket/langsmith/resources.py +12 -0
- evalmarket/langsmith/validation.py +86 -0
- evalmarket_cli-0.1.0.dist-info/METADATA +69 -0
- evalmarket_cli-0.1.0.dist-info/RECORD +24 -0
- evalmarket_cli-0.1.0.dist-info/WHEEL +4 -0
- evalmarket_cli-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Braintrust models for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
from evalmarket.braintrust.resources import model_label, resource_id
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def braintrust_model_choices(client, project):
|
|
7
|
+
"""Use visible configuration first; catalogs are incomplete, not access probes."""
|
|
8
|
+
choices, incomplete = {}, False
|
|
9
|
+
try:
|
|
10
|
+
org_id = resource_id({"id": project.get("org_id")})
|
|
11
|
+
provider_records = list(client.objects("ai_secret"))
|
|
12
|
+
except (ValueError, OSError):
|
|
13
|
+
return [], True
|
|
14
|
+
for record in provider_records:
|
|
15
|
+
if record.get("org_id") != org_id:
|
|
16
|
+
continue
|
|
17
|
+
provider = record.get("name")
|
|
18
|
+
metadata = record.get("metadata") or {}
|
|
19
|
+
if not model_label(provider) or not isinstance(metadata, dict):
|
|
20
|
+
incomplete = True
|
|
21
|
+
continue
|
|
22
|
+
custom = metadata.get("customModels") or {}
|
|
23
|
+
configured = metadata.get("models") or []
|
|
24
|
+
excluded = metadata.get("excludeDefaultModels")
|
|
25
|
+
if (
|
|
26
|
+
not isinstance(custom, dict)
|
|
27
|
+
or not isinstance(configured, list)
|
|
28
|
+
or (excluded is not None and type(excluded) is not bool)
|
|
29
|
+
):
|
|
30
|
+
incomplete = True
|
|
31
|
+
continue
|
|
32
|
+
# Custom model flavor is authoritative; do not offer embedding/realtime
|
|
33
|
+
# definitions as answering/judge models, even if listed in models too.
|
|
34
|
+
non_chat = {
|
|
35
|
+
name
|
|
36
|
+
for name, spec in custom.items()
|
|
37
|
+
if not isinstance(spec, dict) or spec.get("flavor") != "chat"
|
|
38
|
+
}
|
|
39
|
+
for name in custom:
|
|
40
|
+
if model_label(name) and name not in non_chat:
|
|
41
|
+
choices[name, provider] = "configured"
|
|
42
|
+
if excluded is not True:
|
|
43
|
+
for name in configured:
|
|
44
|
+
if model_label(name) and name not in non_chat:
|
|
45
|
+
choices[name, provider] = "configured"
|
|
46
|
+
elif not model_label(name):
|
|
47
|
+
incomplete = True
|
|
48
|
+
if configured or excluded is True:
|
|
49
|
+
continue
|
|
50
|
+
# Only unrestricted providers need an upstream catalog. No seller endpoint
|
|
51
|
+
# is followed, and no provider credential or inference request is needed.
|
|
52
|
+
try:
|
|
53
|
+
catalog = client.request(
|
|
54
|
+
"GET", "/v1/proxy/models", _model_scope=(resource_id(project), provider)
|
|
55
|
+
)
|
|
56
|
+
entries = catalog.get("data") if isinstance(catalog, dict) else None
|
|
57
|
+
if not isinstance(entries, list):
|
|
58
|
+
raise ValueError("Invalid model catalog.")
|
|
59
|
+
for entry in entries:
|
|
60
|
+
if not isinstance(entry, dict):
|
|
61
|
+
incomplete = True
|
|
62
|
+
continue
|
|
63
|
+
name = entry.get("id")
|
|
64
|
+
if (
|
|
65
|
+
model_label(name)
|
|
66
|
+
and name not in non_chat
|
|
67
|
+
and entry.get("flavor", "chat") == "chat"
|
|
68
|
+
):
|
|
69
|
+
choices.setdefault((name, provider), "provider catalog")
|
|
70
|
+
if not entries or catalog.get("has_more") or catalog.get("next_page_token"):
|
|
71
|
+
incomplete = True
|
|
72
|
+
except (ValueError, OSError):
|
|
73
|
+
incomplete = True
|
|
74
|
+
return [
|
|
75
|
+
(model, provider, source)
|
|
76
|
+
for (model, provider), source in sorted(
|
|
77
|
+
choices.items(),
|
|
78
|
+
key=lambda item: (item[0][0].casefold(), item[0][1].casefold()),
|
|
79
|
+
)
|
|
80
|
+
], incomplete
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def choose_braintrust_models(client, project, functions, interactive):
|
|
84
|
+
pending = [f for f in functions if f["function_data"]["type"] == "prompt"]
|
|
85
|
+
if not pending:
|
|
86
|
+
return
|
|
87
|
+
print("\nChoose models")
|
|
88
|
+
print(
|
|
89
|
+
" Select a model/provider from your account for each prompt task and LLM scorer."
|
|
90
|
+
)
|
|
91
|
+
print(" Discovery does not run models. Prompts and parameters stay unchanged.")
|
|
92
|
+
try:
|
|
93
|
+
choices, incomplete = braintrust_model_choices(client, project)
|
|
94
|
+
except KeyboardInterrupt:
|
|
95
|
+
raise ValueError("Model selection cancelled; no writes were made.") from None
|
|
96
|
+
print(
|
|
97
|
+
" Choices come from visible organization configuration or provider catalogs."
|
|
98
|
+
)
|
|
99
|
+
print(
|
|
100
|
+
" Project-only providers/overrides may be missing; chat support and access are not tested."
|
|
101
|
+
)
|
|
102
|
+
if incomplete:
|
|
103
|
+
print(
|
|
104
|
+
" Model discovery is incomplete; only discovered choices can be selected."
|
|
105
|
+
)
|
|
106
|
+
if not choices:
|
|
107
|
+
raise ValueError(
|
|
108
|
+
"No model/provider choices could be discovered for this account/project. Check your Braintrust provider configuration and API-key access, then rerun. No writes were made."
|
|
109
|
+
)
|
|
110
|
+
try:
|
|
111
|
+
for definition in pending:
|
|
112
|
+
options = definition["prompt_data"]["options"]
|
|
113
|
+
print(f"\n{definition['function_type'].title()}: {definition['name']!r}")
|
|
114
|
+
# Only buyer CLI values can be present: package settings were discarded.
|
|
115
|
+
candidates = [
|
|
116
|
+
pair
|
|
117
|
+
for pair in choices
|
|
118
|
+
if ("model" not in options or pair[0] == options["model"])
|
|
119
|
+
and (
|
|
120
|
+
"endpoint_name" not in options
|
|
121
|
+
or pair[1] == options["endpoint_name"]
|
|
122
|
+
)
|
|
123
|
+
]
|
|
124
|
+
if not candidates:
|
|
125
|
+
raise ValueError(
|
|
126
|
+
f"No discovered model/provider matches the buyer CLI selection for {definition['slug']!r}. Check --model/--provider or rerun without those flags to choose from your account. No writes were made."
|
|
127
|
+
)
|
|
128
|
+
if "model" in options and len(candidates) == 1:
|
|
129
|
+
model, provider, _ = candidates[0]
|
|
130
|
+
options.update(model=model, endpoint_name=provider)
|
|
131
|
+
print(f" Selected: {model!r} via {provider!r}")
|
|
132
|
+
continue
|
|
133
|
+
if not interactive:
|
|
134
|
+
raise ValueError(
|
|
135
|
+
f"A unique account model/provider selection is required for {definition['slug']!r}. Supply --model and, if ambiguous, --provider for each prompt task/LLM scorer, or run interactively. No writes were made."
|
|
136
|
+
)
|
|
137
|
+
offset = 0
|
|
138
|
+
while True:
|
|
139
|
+
for index, (model, provider, source) in enumerate(
|
|
140
|
+
candidates[offset : offset + 20], offset + 1
|
|
141
|
+
):
|
|
142
|
+
print(f" {index}. {model!r} via {provider!r} ({source})")
|
|
143
|
+
if len(candidates) > 20:
|
|
144
|
+
print(
|
|
145
|
+
f" Showing {offset + 1}-{min(offset + 20, len(candidates))} of {len(candidates)}; n = next, p = previous."
|
|
146
|
+
)
|
|
147
|
+
choice = input("Model choice [number required, q cancel]: ").strip()
|
|
148
|
+
if choice.lower() == "q":
|
|
149
|
+
raise ValueError("Import cancelled; no writes were made.")
|
|
150
|
+
if choice.lower() in {"n", "p"} and len(candidates) > 20:
|
|
151
|
+
offset = (
|
|
152
|
+
min(offset + 20, ((len(candidates) - 1) // 20) * 20)
|
|
153
|
+
if choice.lower() == "n"
|
|
154
|
+
else max(0, offset - 20)
|
|
155
|
+
)
|
|
156
|
+
continue
|
|
157
|
+
selected = {
|
|
158
|
+
str(index): pair for index, pair in enumerate(candidates, 1)
|
|
159
|
+
}.get(choice.lstrip("0"))
|
|
160
|
+
if selected is None:
|
|
161
|
+
print(
|
|
162
|
+
"Choose a listed model number or q to cancel; a model selection is required."
|
|
163
|
+
)
|
|
164
|
+
continue
|
|
165
|
+
model, provider, _ = selected
|
|
166
|
+
options.update(model=model, endpoint_name=provider)
|
|
167
|
+
break
|
|
168
|
+
except (EOFError, KeyboardInterrupt):
|
|
169
|
+
raise ValueError("Model selection cancelled; no writes were made.") from None
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Braintrust resources for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def resource_id(record):
|
|
7
|
+
try:
|
|
8
|
+
return str(UUID(record["id"]))
|
|
9
|
+
except (KeyError, TypeError, ValueError, AttributeError):
|
|
10
|
+
raise ValueError(
|
|
11
|
+
"Braintrust returned no valid resource ID; inspect the project before retrying."
|
|
12
|
+
) from None
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def model_label(value):
|
|
16
|
+
return (
|
|
17
|
+
isinstance(value, str)
|
|
18
|
+
and 0 < len(value) <= 512
|
|
19
|
+
and value.strip() == value
|
|
20
|
+
and value.isprintable()
|
|
21
|
+
)
|
|
@@ -0,0 +1,368 @@
|
|
|
1
|
+
"""Braintrust validation for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
from evalmarket.json_data import MAX_NATIVE_FILE_BYTES, native_json
|
|
7
|
+
|
|
8
|
+
NUMERIC_PARAMS = {
|
|
9
|
+
"temperature",
|
|
10
|
+
"top_p",
|
|
11
|
+
"top_k",
|
|
12
|
+
"max_tokens",
|
|
13
|
+
"max_completion_tokens",
|
|
14
|
+
"frequency_penalty",
|
|
15
|
+
"presence_penalty",
|
|
16
|
+
"n",
|
|
17
|
+
"reasoning_budget",
|
|
18
|
+
"max_tokens_to_sample",
|
|
19
|
+
"maxOutputTokens",
|
|
20
|
+
"topP",
|
|
21
|
+
"topK",
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
BOOL_PARAMS = {"use_cache", "reasoning_enabled"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def braintrust_package(read, *, discard_legacy_models=False):
|
|
29
|
+
"""Pure structural validation. read(name) returns bytes; no seller execution.
|
|
30
|
+
|
|
31
|
+
Shared by the ZIP checker and local helper. Errors contain field paths,
|
|
32
|
+
never case values, prompt/code contents, or credentials.
|
|
33
|
+
Only the buyer helper discards account settings from older packages; new
|
|
34
|
+
submissions must omit them. Neither path changes source files.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def fail(path, message):
|
|
38
|
+
raise ValueError(f"braintrust/{path}: {message}")
|
|
39
|
+
|
|
40
|
+
def obj(value, allowed, required, path):
|
|
41
|
+
if not isinstance(value, dict):
|
|
42
|
+
fail(path, "must be an object.")
|
|
43
|
+
if set(value) - set(allowed):
|
|
44
|
+
fail(
|
|
45
|
+
path, "contains unsupported fields; consult the allowed fields in Docs."
|
|
46
|
+
)
|
|
47
|
+
if set(required) - set(value):
|
|
48
|
+
fail(
|
|
49
|
+
path,
|
|
50
|
+
"missing required fields: "
|
|
51
|
+
+ ", ".join(sorted(set(required) - set(value))),
|
|
52
|
+
)
|
|
53
|
+
return value
|
|
54
|
+
|
|
55
|
+
def string(value, path):
|
|
56
|
+
if not isinstance(value, str) or not value.strip():
|
|
57
|
+
fail(path, "must be a nonempty string.")
|
|
58
|
+
|
|
59
|
+
def strings(value, path):
|
|
60
|
+
if not isinstance(value, list) or any(not isinstance(v, str) for v in value):
|
|
61
|
+
fail(path, "must be an array of strings.")
|
|
62
|
+
|
|
63
|
+
loaded = {}
|
|
64
|
+
for name in ("README.md", "dataset.json", "tasks.json", "scorers.json"):
|
|
65
|
+
try:
|
|
66
|
+
raw = read(name)
|
|
67
|
+
if not 0 < len(raw) <= MAX_NATIVE_FILE_BYTES:
|
|
68
|
+
fail(name, "must be a nonempty file up to 5 MiB.")
|
|
69
|
+
text = raw.decode("utf-8-sig")
|
|
70
|
+
if not text.strip():
|
|
71
|
+
fail(name, "must be nonempty UTF-8 text.")
|
|
72
|
+
loaded[name] = native_json(text) if name.endswith(".json") else text
|
|
73
|
+
except (OSError, KeyError):
|
|
74
|
+
if name == "tasks.json":
|
|
75
|
+
fail(
|
|
76
|
+
name,
|
|
77
|
+
"required task setup is missing or unreadable. Ask the seller for a complete updated package; older compatibility checks did not require tasks.",
|
|
78
|
+
)
|
|
79
|
+
fail(name, "required file is missing or unreadable.")
|
|
80
|
+
except (UnicodeError, json.JSONDecodeError, RecursionError):
|
|
81
|
+
fail(name, "must contain valid UTF-8 text and JSON where applicable.")
|
|
82
|
+
except ValueError as error:
|
|
83
|
+
if str(error).startswith("braintrust/"):
|
|
84
|
+
raise
|
|
85
|
+
fail(
|
|
86
|
+
name,
|
|
87
|
+
"invalid JSON: duplicate keys or non-finite numbers are not allowed.",
|
|
88
|
+
)
|
|
89
|
+
cases, tasks, scorers = (
|
|
90
|
+
loaded["dataset.json"],
|
|
91
|
+
loaded["tasks.json"],
|
|
92
|
+
loaded["scorers.json"],
|
|
93
|
+
)
|
|
94
|
+
for name, values in (
|
|
95
|
+
("dataset.json", cases),
|
|
96
|
+
("tasks.json", tasks),
|
|
97
|
+
("scorers.json", scorers),
|
|
98
|
+
):
|
|
99
|
+
if not isinstance(values, list) or not values:
|
|
100
|
+
fail(name, "must be a nonempty array.")
|
|
101
|
+
ids = set()
|
|
102
|
+
for index, case in enumerate(cases):
|
|
103
|
+
path = f"dataset.json[{index}]"
|
|
104
|
+
obj(case, {"input", "id", "expected", "metadata", "tags"}, {"input"}, path)
|
|
105
|
+
if "id" in case:
|
|
106
|
+
string(case["id"], path + ".id")
|
|
107
|
+
if case["id"] in ids:
|
|
108
|
+
fail(path + ".id", "duplicate supplied ID.")
|
|
109
|
+
ids.add(case["id"])
|
|
110
|
+
if "metadata" in case and not isinstance(case["metadata"], dict):
|
|
111
|
+
fail(path + ".metadata", "must be an object.")
|
|
112
|
+
if "tags" in case:
|
|
113
|
+
strings(case["tags"], path + ".tags")
|
|
114
|
+
slugs = set()
|
|
115
|
+
functions = [
|
|
116
|
+
(filename, index, kind, definition)
|
|
117
|
+
for filename, kind, definitions in (
|
|
118
|
+
("tasks.json", "task", tasks),
|
|
119
|
+
("scorers.json", "scorer", scorers),
|
|
120
|
+
)
|
|
121
|
+
for index, definition in enumerate(definitions)
|
|
122
|
+
]
|
|
123
|
+
for filename, index, kind, scorer in functions:
|
|
124
|
+
path = f"{filename}[{index}]"
|
|
125
|
+
obj(
|
|
126
|
+
scorer,
|
|
127
|
+
{
|
|
128
|
+
"name",
|
|
129
|
+
"slug",
|
|
130
|
+
"description",
|
|
131
|
+
"tags",
|
|
132
|
+
"function_type",
|
|
133
|
+
"function_data",
|
|
134
|
+
"function_schema",
|
|
135
|
+
"prompt_data",
|
|
136
|
+
},
|
|
137
|
+
{"name", "slug", "function_type", "function_data"},
|
|
138
|
+
path,
|
|
139
|
+
)
|
|
140
|
+
string(scorer["name"], path + ".name")
|
|
141
|
+
slug = scorer["slug"]
|
|
142
|
+
if (
|
|
143
|
+
not isinstance(slug, str)
|
|
144
|
+
or not re.fullmatch(r"[a-z0-9]+(?:[-_][a-z0-9]+)*", slug)
|
|
145
|
+
or len(slug) > 80
|
|
146
|
+
):
|
|
147
|
+
fail(
|
|
148
|
+
path + ".slug",
|
|
149
|
+
"use 1–80 lowercase letters/digits with single hyphen/underscore separators.",
|
|
150
|
+
)
|
|
151
|
+
if slug in slugs:
|
|
152
|
+
fail(
|
|
153
|
+
path + ".slug",
|
|
154
|
+
"duplicate task/scorer slug; slugs must be unique across both files.",
|
|
155
|
+
)
|
|
156
|
+
slugs.add(slug)
|
|
157
|
+
if scorer["function_type"] != kind:
|
|
158
|
+
fail(path + ".function_type", f"must be {kind}.")
|
|
159
|
+
if "description" in scorer and not isinstance(scorer["description"], str):
|
|
160
|
+
fail(path + ".description", "must be a string.")
|
|
161
|
+
if "tags" in scorer:
|
|
162
|
+
strings(scorer["tags"], path + ".tags")
|
|
163
|
+
if "function_schema" in scorer:
|
|
164
|
+
schema = obj(
|
|
165
|
+
scorer["function_schema"],
|
|
166
|
+
{"parameters", "returns"},
|
|
167
|
+
set(),
|
|
168
|
+
path + ".function_schema",
|
|
169
|
+
)
|
|
170
|
+
for value in schema.values():
|
|
171
|
+
if not isinstance(value, (dict, bool)):
|
|
172
|
+
fail(
|
|
173
|
+
path + ".function_schema",
|
|
174
|
+
"parameters/returns must be JSON schema objects or booleans.",
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
# Schemas are data; external references cannot be resolved by this importer.
|
|
178
|
+
def refs(value, path=path):
|
|
179
|
+
if isinstance(value, dict):
|
|
180
|
+
for key, child in value.items():
|
|
181
|
+
if key in {"$ref", "$dynamicRef"} and (
|
|
182
|
+
not isinstance(child, str) or not child.startswith("#")
|
|
183
|
+
):
|
|
184
|
+
fail(
|
|
185
|
+
path + ".function_schema",
|
|
186
|
+
"external schema references are unsupported.",
|
|
187
|
+
)
|
|
188
|
+
refs(child)
|
|
189
|
+
elif isinstance(value, list):
|
|
190
|
+
for child in value:
|
|
191
|
+
refs(child)
|
|
192
|
+
|
|
193
|
+
refs(schema)
|
|
194
|
+
data = obj(
|
|
195
|
+
scorer["function_data"], {"type", "data"}, {"type"}, path + ".function_data"
|
|
196
|
+
)
|
|
197
|
+
if data["type"] == "code":
|
|
198
|
+
if "prompt_data" in scorer:
|
|
199
|
+
fail(path, "code functions cannot include prompt_data.")
|
|
200
|
+
inline = obj(
|
|
201
|
+
data.get("data"),
|
|
202
|
+
{"type", "runtime_context", "code"},
|
|
203
|
+
{"type", "runtime_context", "code"},
|
|
204
|
+
path + ".function_data.data",
|
|
205
|
+
)
|
|
206
|
+
if inline["type"] != "inline":
|
|
207
|
+
fail(
|
|
208
|
+
path + ".function_data.data.type", "only inline code is supported."
|
|
209
|
+
)
|
|
210
|
+
runtime = obj(
|
|
211
|
+
inline["runtime_context"],
|
|
212
|
+
{"runtime", "version"},
|
|
213
|
+
{"runtime", "version"},
|
|
214
|
+
path + ".runtime_context",
|
|
215
|
+
)
|
|
216
|
+
if (runtime["runtime"], runtime["version"]) not in (
|
|
217
|
+
("node", "22"),
|
|
218
|
+
("python", "3.12"),
|
|
219
|
+
):
|
|
220
|
+
fail(
|
|
221
|
+
path + ".runtime_context",
|
|
222
|
+
"supported runtimes are node 22 and python 3.12.",
|
|
223
|
+
)
|
|
224
|
+
string(inline["code"], path + ".code")
|
|
225
|
+
elif data["type"] == "prompt":
|
|
226
|
+
if set(data) != {"type"}:
|
|
227
|
+
fail(
|
|
228
|
+
path + ".function_data", "prompt function_data contains only type."
|
|
229
|
+
)
|
|
230
|
+
prompt = obj(
|
|
231
|
+
scorer.get("prompt_data"),
|
|
232
|
+
{"prompt", "options", "template_format"}
|
|
233
|
+
| ({"parser"} if kind == "scorer" else set()),
|
|
234
|
+
{"prompt", "options"} | ({"parser"} if kind == "scorer" else set()),
|
|
235
|
+
path + ".prompt_data",
|
|
236
|
+
)
|
|
237
|
+
if "template_format" in prompt and prompt["template_format"] not in (
|
|
238
|
+
"mustache",
|
|
239
|
+
"nunjucks",
|
|
240
|
+
"none",
|
|
241
|
+
):
|
|
242
|
+
fail(path + ".template_format", "use mustache, nunjucks, or none.")
|
|
243
|
+
if (
|
|
244
|
+
kind == "task"
|
|
245
|
+
and prompt.get("template_format", "mustache") != "mustache"
|
|
246
|
+
):
|
|
247
|
+
fail(
|
|
248
|
+
path + ".template_format",
|
|
249
|
+
"prompt tasks use mustache with input-only bindings.",
|
|
250
|
+
)
|
|
251
|
+
chat = obj(
|
|
252
|
+
prompt["prompt"],
|
|
253
|
+
{"type", "messages"},
|
|
254
|
+
{"type", "messages"},
|
|
255
|
+
path + ".prompt",
|
|
256
|
+
)
|
|
257
|
+
if (
|
|
258
|
+
chat["type"] != "chat"
|
|
259
|
+
or not isinstance(chat["messages"], list)
|
|
260
|
+
or not chat["messages"]
|
|
261
|
+
):
|
|
262
|
+
fail(path + ".prompt", "must be chat with a nonempty messages array.")
|
|
263
|
+
for i, message in enumerate(chat["messages"]):
|
|
264
|
+
mp = path + f".prompt.messages[{i}]"
|
|
265
|
+
obj(message, {"role", "content", "name"}, {"role", "content"}, mp)
|
|
266
|
+
if message["role"] not in ("system", "developer", "user", "assistant"):
|
|
267
|
+
fail(
|
|
268
|
+
mp + ".role",
|
|
269
|
+
"use system, developer, user, or assistant; tools are unsupported.",
|
|
270
|
+
)
|
|
271
|
+
string(message["content"], mp + ".content")
|
|
272
|
+
if kind == "task":
|
|
273
|
+
# Scan once: unmatched seller-supplied braces must not cause
|
|
274
|
+
# a backtracking regex to repeatedly scan a multi-MiB prompt.
|
|
275
|
+
content, offset = message["content"], 0
|
|
276
|
+
while (start := content.find("{{", offset)) != -1:
|
|
277
|
+
width = 3 if content.startswith("{{{", start) else 2
|
|
278
|
+
end = content.find("}" * width, start + width)
|
|
279
|
+
if end == -1:
|
|
280
|
+
fail(mp + ".content", "malformed task input binding.")
|
|
281
|
+
binding = content[start + width : end].strip()
|
|
282
|
+
if not re.fullmatch(r"input(?:\.[A-Za-z0-9_-]+)*", binding):
|
|
283
|
+
fail(
|
|
284
|
+
mp + ".content",
|
|
285
|
+
"task bindings may reference only input or input.field; expected answers, outputs, and grading metadata are not task inputs.",
|
|
286
|
+
)
|
|
287
|
+
offset = end + width
|
|
288
|
+
if "name" in message:
|
|
289
|
+
string(message["name"], mp + ".name")
|
|
290
|
+
options = obj(
|
|
291
|
+
prompt["options"],
|
|
292
|
+
{"model", "params", "position", "endpoint_name"},
|
|
293
|
+
set(),
|
|
294
|
+
path + ".options",
|
|
295
|
+
)
|
|
296
|
+
for key in ("model", "endpoint_name"):
|
|
297
|
+
if key in options:
|
|
298
|
+
if not discard_legacy_models:
|
|
299
|
+
fail(
|
|
300
|
+
path + ".options." + key,
|
|
301
|
+
"remove this field; the importer selects model/provider configuration from the buyer's account.",
|
|
302
|
+
)
|
|
303
|
+
# Older published files stay byte-for-byte intact. Their
|
|
304
|
+
# account-specific settings are never used or displayed.
|
|
305
|
+
options.pop(key)
|
|
306
|
+
if "position" in options:
|
|
307
|
+
string(options["position"], path + ".options.position")
|
|
308
|
+
if "params" in options:
|
|
309
|
+
params = obj(
|
|
310
|
+
options["params"],
|
|
311
|
+
NUMERIC_PARAMS
|
|
312
|
+
| BOOL_PARAMS
|
|
313
|
+
| {"stop", "stop_sequences", "reasoning_effort", "verbosity"},
|
|
314
|
+
set(),
|
|
315
|
+
path + ".options.params",
|
|
316
|
+
)
|
|
317
|
+
for key, value in params.items():
|
|
318
|
+
pp = path + ".options.params." + key
|
|
319
|
+
if key in NUMERIC_PARAMS and type(value) not in (int, float):
|
|
320
|
+
fail(pp, "must be a finite number.")
|
|
321
|
+
if key in BOOL_PARAMS and not isinstance(value, bool):
|
|
322
|
+
fail(pp, "must be boolean.")
|
|
323
|
+
if key in ("stop", "stop_sequences"):
|
|
324
|
+
strings(value, pp)
|
|
325
|
+
if key == "reasoning_effort" and value not in (
|
|
326
|
+
"none",
|
|
327
|
+
"minimal",
|
|
328
|
+
"low",
|
|
329
|
+
"medium",
|
|
330
|
+
"high",
|
|
331
|
+
):
|
|
332
|
+
fail(pp, "use none, minimal, low, medium, or high.")
|
|
333
|
+
if key == "verbosity" and value not in ("low", "medium", "high"):
|
|
334
|
+
fail(pp, "use low, medium, or high.")
|
|
335
|
+
if kind == "task":
|
|
336
|
+
continue
|
|
337
|
+
parser = obj(
|
|
338
|
+
prompt["parser"],
|
|
339
|
+
{"type", "use_cot", "choice_scores", "allow_skip"},
|
|
340
|
+
{"type", "use_cot", "choice_scores"},
|
|
341
|
+
path + ".parser",
|
|
342
|
+
)
|
|
343
|
+
if parser["type"] != "llm_classifier":
|
|
344
|
+
fail(path + ".parser.type", "must be llm_classifier.")
|
|
345
|
+
for key in ("use_cot", "allow_skip"):
|
|
346
|
+
if key in parser and not isinstance(parser[key], bool):
|
|
347
|
+
fail(path + ".parser." + key, "must be boolean.")
|
|
348
|
+
choices = parser["choice_scores"]
|
|
349
|
+
if (
|
|
350
|
+
not isinstance(choices, dict)
|
|
351
|
+
or not choices
|
|
352
|
+
or any(
|
|
353
|
+
not key.strip()
|
|
354
|
+
or type(value) not in (int, float)
|
|
355
|
+
or not 0 <= value <= 1
|
|
356
|
+
for key, value in choices.items()
|
|
357
|
+
)
|
|
358
|
+
):
|
|
359
|
+
fail(
|
|
360
|
+
path + ".parser.choice_scores",
|
|
361
|
+
"use a nonempty choice-to-score mapping with finite numbers from 0 to 1.",
|
|
362
|
+
)
|
|
363
|
+
else:
|
|
364
|
+
fail(
|
|
365
|
+
path + ".function_data.type",
|
|
366
|
+
"only code and prompt tasks/scorers are supported.",
|
|
367
|
+
)
|
|
368
|
+
return cases, tasks, scorers
|
evalmarket/cli.py
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
"""Command-line parsing and user-facing import errors."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import subprocess
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from evalmarket._version import __version__
|
|
9
|
+
from evalmarket.braintrust.experiments import ExperimentStartError
|
|
10
|
+
from evalmarket.braintrust.importer import import_braintrust
|
|
11
|
+
from evalmarket.langsmith.importer import import_package
|
|
12
|
+
from evalmarket.langsmith.playground import PlaygroundSetupError
|
|
13
|
+
|
|
14
|
+
DESCRIPTION = (
|
|
15
|
+
"Import native eval files, then optionally start Braintrust or open LangSmith."
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def add_import_arguments(parser):
|
|
20
|
+
parser.add_argument(
|
|
21
|
+
"folder",
|
|
22
|
+
help="Extracted platform folder: braintrust or langchain (or a path to it)",
|
|
23
|
+
)
|
|
24
|
+
parser.add_argument("--project-id", help="Braintrust destination project UUID")
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"--yes", action="store_true", help="Confirm the displayed Braintrust import"
|
|
27
|
+
)
|
|
28
|
+
parser.add_argument(
|
|
29
|
+
"--provider",
|
|
30
|
+
action="append",
|
|
31
|
+
help="Braintrust buyer provider selection: function-slug=discovered-provider-name (required when the model is ambiguous)",
|
|
32
|
+
)
|
|
33
|
+
parser.add_argument(
|
|
34
|
+
"--model",
|
|
35
|
+
action="append",
|
|
36
|
+
help="Braintrust buyer model selection: function-slug=discovered-model-name (required per prompt task/LLM scorer in unattended imports)",
|
|
37
|
+
)
|
|
38
|
+
parser.add_argument(
|
|
39
|
+
"--dataset",
|
|
40
|
+
help="Optional dataset name; defaults to the evaluator name plus a unique timestamp",
|
|
41
|
+
)
|
|
42
|
+
parser.add_argument(
|
|
43
|
+
"--profile",
|
|
44
|
+
help="Existing LangSmith CLI profile; otherwise use the CLI's current authentication",
|
|
45
|
+
)
|
|
46
|
+
parser.add_argument(
|
|
47
|
+
"--model-config",
|
|
48
|
+
help="Optional existing native model JSON; otherwise choose a saved LangSmith model interactively",
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def main(argv=None):
|
|
53
|
+
parser = argparse.ArgumentParser(prog="evalmarket", description=DESCRIPTION)
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--version", action="version", version=f"evalmarket {__version__}"
|
|
56
|
+
)
|
|
57
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
58
|
+
importer = commands.add_parser(
|
|
59
|
+
"import", help="Import a native platform folder", description=DESCRIPTION
|
|
60
|
+
)
|
|
61
|
+
add_import_arguments(importer)
|
|
62
|
+
return run_import(parser.parse_args(argv))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def run_import(args):
|
|
66
|
+
try:
|
|
67
|
+
if Path(args.folder).name == "braintrust":
|
|
68
|
+
if args.dataset or args.profile or args.model_config:
|
|
69
|
+
raise ValueError(
|
|
70
|
+
"--dataset, --profile and --model-config are LangSmith-only options."
|
|
71
|
+
)
|
|
72
|
+
import_braintrust(
|
|
73
|
+
args.folder, args.project_id, args.yes, args.model, args.provider
|
|
74
|
+
)
|
|
75
|
+
else:
|
|
76
|
+
if args.project_id or args.yes or args.model or args.provider:
|
|
77
|
+
raise ValueError("Braintrust options require a braintrust/ folder.")
|
|
78
|
+
import_package(args.folder, args.dataset, args.profile, args.model_config)
|
|
79
|
+
except PlaygroundSetupError as error:
|
|
80
|
+
print(
|
|
81
|
+
f"\nImport complete; Playground setup needs attention.\n{error}",
|
|
82
|
+
file=sys.stderr,
|
|
83
|
+
)
|
|
84
|
+
return 1
|
|
85
|
+
except ExperimentStartError as error:
|
|
86
|
+
print(
|
|
87
|
+
f"\nImport complete; experiment start needs attention.\n{error}",
|
|
88
|
+
file=sys.stderr,
|
|
89
|
+
)
|
|
90
|
+
return 1
|
|
91
|
+
except (
|
|
92
|
+
OSError,
|
|
93
|
+
ValueError,
|
|
94
|
+
subprocess.CalledProcessError,
|
|
95
|
+
KeyboardInterrupt,
|
|
96
|
+
EOFError,
|
|
97
|
+
) as error:
|
|
98
|
+
print(
|
|
99
|
+
f"Import stopped: {error}\nIf an upload started, inspect your destination workspace for partial resources before retrying; the CLI does not retry or replace existing resources.",
|
|
100
|
+
file=sys.stderr,
|
|
101
|
+
)
|
|
102
|
+
return 1
|
|
103
|
+
return 0
|