evalmarket-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,169 @@
1
+ """Braintrust models for native evaluation imports."""
2
+
3
+ from evalmarket.braintrust.resources import model_label, resource_id
4
+
5
+
6
+ def braintrust_model_choices(client, project):
7
+ """Use visible configuration first; catalogs are incomplete, not access probes."""
8
+ choices, incomplete = {}, False
9
+ try:
10
+ org_id = resource_id({"id": project.get("org_id")})
11
+ provider_records = list(client.objects("ai_secret"))
12
+ except (ValueError, OSError):
13
+ return [], True
14
+ for record in provider_records:
15
+ if record.get("org_id") != org_id:
16
+ continue
17
+ provider = record.get("name")
18
+ metadata = record.get("metadata") or {}
19
+ if not model_label(provider) or not isinstance(metadata, dict):
20
+ incomplete = True
21
+ continue
22
+ custom = metadata.get("customModels") or {}
23
+ configured = metadata.get("models") or []
24
+ excluded = metadata.get("excludeDefaultModels")
25
+ if (
26
+ not isinstance(custom, dict)
27
+ or not isinstance(configured, list)
28
+ or (excluded is not None and type(excluded) is not bool)
29
+ ):
30
+ incomplete = True
31
+ continue
32
+ # Custom model flavor is authoritative; do not offer embedding/realtime
33
+ # definitions as answering/judge models, even if listed in models too.
34
+ non_chat = {
35
+ name
36
+ for name, spec in custom.items()
37
+ if not isinstance(spec, dict) or spec.get("flavor") != "chat"
38
+ }
39
+ for name in custom:
40
+ if model_label(name) and name not in non_chat:
41
+ choices[name, provider] = "configured"
42
+ if excluded is not True:
43
+ for name in configured:
44
+ if model_label(name) and name not in non_chat:
45
+ choices[name, provider] = "configured"
46
+ elif not model_label(name):
47
+ incomplete = True
48
+ if configured or excluded is True:
49
+ continue
50
+ # Only unrestricted providers need an upstream catalog. No seller endpoint
51
+ # is followed, and no provider credential or inference request is needed.
52
+ try:
53
+ catalog = client.request(
54
+ "GET", "/v1/proxy/models", _model_scope=(resource_id(project), provider)
55
+ )
56
+ entries = catalog.get("data") if isinstance(catalog, dict) else None
57
+ if not isinstance(entries, list):
58
+ raise ValueError("Invalid model catalog.")
59
+ for entry in entries:
60
+ if not isinstance(entry, dict):
61
+ incomplete = True
62
+ continue
63
+ name = entry.get("id")
64
+ if (
65
+ model_label(name)
66
+ and name not in non_chat
67
+ and entry.get("flavor", "chat") == "chat"
68
+ ):
69
+ choices.setdefault((name, provider), "provider catalog")
70
+ if not entries or catalog.get("has_more") or catalog.get("next_page_token"):
71
+ incomplete = True
72
+ except (ValueError, OSError):
73
+ incomplete = True
74
+ return [
75
+ (model, provider, source)
76
+ for (model, provider), source in sorted(
77
+ choices.items(),
78
+ key=lambda item: (item[0][0].casefold(), item[0][1].casefold()),
79
+ )
80
+ ], incomplete
81
+
82
+
83
+ def choose_braintrust_models(client, project, functions, interactive):
84
+ pending = [f for f in functions if f["function_data"]["type"] == "prompt"]
85
+ if not pending:
86
+ return
87
+ print("\nChoose models")
88
+ print(
89
+ " Select a model/provider from your account for each prompt task and LLM scorer."
90
+ )
91
+ print(" Discovery does not run models. Prompts and parameters stay unchanged.")
92
+ try:
93
+ choices, incomplete = braintrust_model_choices(client, project)
94
+ except KeyboardInterrupt:
95
+ raise ValueError("Model selection cancelled; no writes were made.") from None
96
+ print(
97
+ " Choices come from visible organization configuration or provider catalogs."
98
+ )
99
+ print(
100
+ " Project-only providers/overrides may be missing; chat support and access are not tested."
101
+ )
102
+ if incomplete:
103
+ print(
104
+ " Model discovery is incomplete; only discovered choices can be selected."
105
+ )
106
+ if not choices:
107
+ raise ValueError(
108
+ "No model/provider choices could be discovered for this account/project. Check your Braintrust provider configuration and API-key access, then rerun. No writes were made."
109
+ )
110
+ try:
111
+ for definition in pending:
112
+ options = definition["prompt_data"]["options"]
113
+ print(f"\n{definition['function_type'].title()}: {definition['name']!r}")
114
+ # Only buyer CLI values can be present: package settings were discarded.
115
+ candidates = [
116
+ pair
117
+ for pair in choices
118
+ if ("model" not in options or pair[0] == options["model"])
119
+ and (
120
+ "endpoint_name" not in options
121
+ or pair[1] == options["endpoint_name"]
122
+ )
123
+ ]
124
+ if not candidates:
125
+ raise ValueError(
126
+ f"No discovered model/provider matches the buyer CLI selection for {definition['slug']!r}. Check --model/--provider or rerun without those flags to choose from your account. No writes were made."
127
+ )
128
+ if "model" in options and len(candidates) == 1:
129
+ model, provider, _ = candidates[0]
130
+ options.update(model=model, endpoint_name=provider)
131
+ print(f" Selected: {model!r} via {provider!r}")
132
+ continue
133
+ if not interactive:
134
+ raise ValueError(
135
+ f"A unique account model/provider selection is required for {definition['slug']!r}. Supply --model and, if ambiguous, --provider for each prompt task/LLM scorer, or run interactively. No writes were made."
136
+ )
137
+ offset = 0
138
+ while True:
139
+ for index, (model, provider, source) in enumerate(
140
+ candidates[offset : offset + 20], offset + 1
141
+ ):
142
+ print(f" {index}. {model!r} via {provider!r} ({source})")
143
+ if len(candidates) > 20:
144
+ print(
145
+ f" Showing {offset + 1}-{min(offset + 20, len(candidates))} of {len(candidates)}; n = next, p = previous."
146
+ )
147
+ choice = input("Model choice [number required, q cancel]: ").strip()
148
+ if choice.lower() == "q":
149
+ raise ValueError("Import cancelled; no writes were made.")
150
+ if choice.lower() in {"n", "p"} and len(candidates) > 20:
151
+ offset = (
152
+ min(offset + 20, ((len(candidates) - 1) // 20) * 20)
153
+ if choice.lower() == "n"
154
+ else max(0, offset - 20)
155
+ )
156
+ continue
157
+ selected = {
158
+ str(index): pair for index, pair in enumerate(candidates, 1)
159
+ }.get(choice.lstrip("0"))
160
+ if selected is None:
161
+ print(
162
+ "Choose a listed model number or q to cancel; a model selection is required."
163
+ )
164
+ continue
165
+ model, provider, _ = selected
166
+ options.update(model=model, endpoint_name=provider)
167
+ break
168
+ except (EOFError, KeyboardInterrupt):
169
+ raise ValueError("Model selection cancelled; no writes were made.") from None
@@ -0,0 +1,21 @@
1
+ """Braintrust resources for native evaluation imports."""
2
+
3
+ from uuid import UUID
4
+
5
+
6
+ def resource_id(record):
7
+ try:
8
+ return str(UUID(record["id"]))
9
+ except (KeyError, TypeError, ValueError, AttributeError):
10
+ raise ValueError(
11
+ "Braintrust returned no valid resource ID; inspect the project before retrying."
12
+ ) from None
13
+
14
+
15
+ def model_label(value):
16
+ return (
17
+ isinstance(value, str)
18
+ and 0 < len(value) <= 512
19
+ and value.strip() == value
20
+ and value.isprintable()
21
+ )
@@ -0,0 +1,368 @@
1
+ """Braintrust validation for native evaluation imports."""
2
+
3
+ import json
4
+ import re
5
+
6
+ from evalmarket.json_data import MAX_NATIVE_FILE_BYTES, native_json
7
+
8
+ NUMERIC_PARAMS = {
9
+ "temperature",
10
+ "top_p",
11
+ "top_k",
12
+ "max_tokens",
13
+ "max_completion_tokens",
14
+ "frequency_penalty",
15
+ "presence_penalty",
16
+ "n",
17
+ "reasoning_budget",
18
+ "max_tokens_to_sample",
19
+ "maxOutputTokens",
20
+ "topP",
21
+ "topK",
22
+ }
23
+
24
+
25
+ BOOL_PARAMS = {"use_cache", "reasoning_enabled"}
26
+
27
+
28
+ def braintrust_package(read, *, discard_legacy_models=False):
29
+ """Pure structural validation. read(name) returns bytes; no seller execution.
30
+
31
+ Shared by the ZIP checker and local helper. Errors contain field paths,
32
+ never case values, prompt/code contents, or credentials.
33
+ Only the buyer helper discards account settings from older packages; new
34
+ submissions must omit them. Neither path changes source files.
35
+ """
36
+
37
+ def fail(path, message):
38
+ raise ValueError(f"braintrust/{path}: {message}")
39
+
40
+ def obj(value, allowed, required, path):
41
+ if not isinstance(value, dict):
42
+ fail(path, "must be an object.")
43
+ if set(value) - set(allowed):
44
+ fail(
45
+ path, "contains unsupported fields; consult the allowed fields in Docs."
46
+ )
47
+ if set(required) - set(value):
48
+ fail(
49
+ path,
50
+ "missing required fields: "
51
+ + ", ".join(sorted(set(required) - set(value))),
52
+ )
53
+ return value
54
+
55
+ def string(value, path):
56
+ if not isinstance(value, str) or not value.strip():
57
+ fail(path, "must be a nonempty string.")
58
+
59
+ def strings(value, path):
60
+ if not isinstance(value, list) or any(not isinstance(v, str) for v in value):
61
+ fail(path, "must be an array of strings.")
62
+
63
+ loaded = {}
64
+ for name in ("README.md", "dataset.json", "tasks.json", "scorers.json"):
65
+ try:
66
+ raw = read(name)
67
+ if not 0 < len(raw) <= MAX_NATIVE_FILE_BYTES:
68
+ fail(name, "must be a nonempty file up to 5 MiB.")
69
+ text = raw.decode("utf-8-sig")
70
+ if not text.strip():
71
+ fail(name, "must be nonempty UTF-8 text.")
72
+ loaded[name] = native_json(text) if name.endswith(".json") else text
73
+ except (OSError, KeyError):
74
+ if name == "tasks.json":
75
+ fail(
76
+ name,
77
+ "required task setup is missing or unreadable. Ask the seller for a complete updated package; older compatibility checks did not require tasks.",
78
+ )
79
+ fail(name, "required file is missing or unreadable.")
80
+ except (UnicodeError, json.JSONDecodeError, RecursionError):
81
+ fail(name, "must contain valid UTF-8 text and JSON where applicable.")
82
+ except ValueError as error:
83
+ if str(error).startswith("braintrust/"):
84
+ raise
85
+ fail(
86
+ name,
87
+ "invalid JSON: duplicate keys or non-finite numbers are not allowed.",
88
+ )
89
+ cases, tasks, scorers = (
90
+ loaded["dataset.json"],
91
+ loaded["tasks.json"],
92
+ loaded["scorers.json"],
93
+ )
94
+ for name, values in (
95
+ ("dataset.json", cases),
96
+ ("tasks.json", tasks),
97
+ ("scorers.json", scorers),
98
+ ):
99
+ if not isinstance(values, list) or not values:
100
+ fail(name, "must be a nonempty array.")
101
+ ids = set()
102
+ for index, case in enumerate(cases):
103
+ path = f"dataset.json[{index}]"
104
+ obj(case, {"input", "id", "expected", "metadata", "tags"}, {"input"}, path)
105
+ if "id" in case:
106
+ string(case["id"], path + ".id")
107
+ if case["id"] in ids:
108
+ fail(path + ".id", "duplicate supplied ID.")
109
+ ids.add(case["id"])
110
+ if "metadata" in case and not isinstance(case["metadata"], dict):
111
+ fail(path + ".metadata", "must be an object.")
112
+ if "tags" in case:
113
+ strings(case["tags"], path + ".tags")
114
+ slugs = set()
115
+ functions = [
116
+ (filename, index, kind, definition)
117
+ for filename, kind, definitions in (
118
+ ("tasks.json", "task", tasks),
119
+ ("scorers.json", "scorer", scorers),
120
+ )
121
+ for index, definition in enumerate(definitions)
122
+ ]
123
+ for filename, index, kind, scorer in functions:
124
+ path = f"{filename}[{index}]"
125
+ obj(
126
+ scorer,
127
+ {
128
+ "name",
129
+ "slug",
130
+ "description",
131
+ "tags",
132
+ "function_type",
133
+ "function_data",
134
+ "function_schema",
135
+ "prompt_data",
136
+ },
137
+ {"name", "slug", "function_type", "function_data"},
138
+ path,
139
+ )
140
+ string(scorer["name"], path + ".name")
141
+ slug = scorer["slug"]
142
+ if (
143
+ not isinstance(slug, str)
144
+ or not re.fullmatch(r"[a-z0-9]+(?:[-_][a-z0-9]+)*", slug)
145
+ or len(slug) > 80
146
+ ):
147
+ fail(
148
+ path + ".slug",
149
+ "use 1–80 lowercase letters/digits with single hyphen/underscore separators.",
150
+ )
151
+ if slug in slugs:
152
+ fail(
153
+ path + ".slug",
154
+ "duplicate task/scorer slug; slugs must be unique across both files.",
155
+ )
156
+ slugs.add(slug)
157
+ if scorer["function_type"] != kind:
158
+ fail(path + ".function_type", f"must be {kind}.")
159
+ if "description" in scorer and not isinstance(scorer["description"], str):
160
+ fail(path + ".description", "must be a string.")
161
+ if "tags" in scorer:
162
+ strings(scorer["tags"], path + ".tags")
163
+ if "function_schema" in scorer:
164
+ schema = obj(
165
+ scorer["function_schema"],
166
+ {"parameters", "returns"},
167
+ set(),
168
+ path + ".function_schema",
169
+ )
170
+ for value in schema.values():
171
+ if not isinstance(value, (dict, bool)):
172
+ fail(
173
+ path + ".function_schema",
174
+ "parameters/returns must be JSON schema objects or booleans.",
175
+ )
176
+
177
+ # Schemas are data; external references cannot be resolved by this importer.
178
+ def refs(value, path=path):
179
+ if isinstance(value, dict):
180
+ for key, child in value.items():
181
+ if key in {"$ref", "$dynamicRef"} and (
182
+ not isinstance(child, str) or not child.startswith("#")
183
+ ):
184
+ fail(
185
+ path + ".function_schema",
186
+ "external schema references are unsupported.",
187
+ )
188
+ refs(child)
189
+ elif isinstance(value, list):
190
+ for child in value:
191
+ refs(child)
192
+
193
+ refs(schema)
194
+ data = obj(
195
+ scorer["function_data"], {"type", "data"}, {"type"}, path + ".function_data"
196
+ )
197
+ if data["type"] == "code":
198
+ if "prompt_data" in scorer:
199
+ fail(path, "code functions cannot include prompt_data.")
200
+ inline = obj(
201
+ data.get("data"),
202
+ {"type", "runtime_context", "code"},
203
+ {"type", "runtime_context", "code"},
204
+ path + ".function_data.data",
205
+ )
206
+ if inline["type"] != "inline":
207
+ fail(
208
+ path + ".function_data.data.type", "only inline code is supported."
209
+ )
210
+ runtime = obj(
211
+ inline["runtime_context"],
212
+ {"runtime", "version"},
213
+ {"runtime", "version"},
214
+ path + ".runtime_context",
215
+ )
216
+ if (runtime["runtime"], runtime["version"]) not in (
217
+ ("node", "22"),
218
+ ("python", "3.12"),
219
+ ):
220
+ fail(
221
+ path + ".runtime_context",
222
+ "supported runtimes are node 22 and python 3.12.",
223
+ )
224
+ string(inline["code"], path + ".code")
225
+ elif data["type"] == "prompt":
226
+ if set(data) != {"type"}:
227
+ fail(
228
+ path + ".function_data", "prompt function_data contains only type."
229
+ )
230
+ prompt = obj(
231
+ scorer.get("prompt_data"),
232
+ {"prompt", "options", "template_format"}
233
+ | ({"parser"} if kind == "scorer" else set()),
234
+ {"prompt", "options"} | ({"parser"} if kind == "scorer" else set()),
235
+ path + ".prompt_data",
236
+ )
237
+ if "template_format" in prompt and prompt["template_format"] not in (
238
+ "mustache",
239
+ "nunjucks",
240
+ "none",
241
+ ):
242
+ fail(path + ".template_format", "use mustache, nunjucks, or none.")
243
+ if (
244
+ kind == "task"
245
+ and prompt.get("template_format", "mustache") != "mustache"
246
+ ):
247
+ fail(
248
+ path + ".template_format",
249
+ "prompt tasks use mustache with input-only bindings.",
250
+ )
251
+ chat = obj(
252
+ prompt["prompt"],
253
+ {"type", "messages"},
254
+ {"type", "messages"},
255
+ path + ".prompt",
256
+ )
257
+ if (
258
+ chat["type"] != "chat"
259
+ or not isinstance(chat["messages"], list)
260
+ or not chat["messages"]
261
+ ):
262
+ fail(path + ".prompt", "must be chat with a nonempty messages array.")
263
+ for i, message in enumerate(chat["messages"]):
264
+ mp = path + f".prompt.messages[{i}]"
265
+ obj(message, {"role", "content", "name"}, {"role", "content"}, mp)
266
+ if message["role"] not in ("system", "developer", "user", "assistant"):
267
+ fail(
268
+ mp + ".role",
269
+ "use system, developer, user, or assistant; tools are unsupported.",
270
+ )
271
+ string(message["content"], mp + ".content")
272
+ if kind == "task":
273
+ # Scan once: unmatched seller-supplied braces must not cause
274
+ # a backtracking regex to repeatedly scan a multi-MiB prompt.
275
+ content, offset = message["content"], 0
276
+ while (start := content.find("{{", offset)) != -1:
277
+ width = 3 if content.startswith("{{{", start) else 2
278
+ end = content.find("}" * width, start + width)
279
+ if end == -1:
280
+ fail(mp + ".content", "malformed task input binding.")
281
+ binding = content[start + width : end].strip()
282
+ if not re.fullmatch(r"input(?:\.[A-Za-z0-9_-]+)*", binding):
283
+ fail(
284
+ mp + ".content",
285
+ "task bindings may reference only input or input.field; expected answers, outputs, and grading metadata are not task inputs.",
286
+ )
287
+ offset = end + width
288
+ if "name" in message:
289
+ string(message["name"], mp + ".name")
290
+ options = obj(
291
+ prompt["options"],
292
+ {"model", "params", "position", "endpoint_name"},
293
+ set(),
294
+ path + ".options",
295
+ )
296
+ for key in ("model", "endpoint_name"):
297
+ if key in options:
298
+ if not discard_legacy_models:
299
+ fail(
300
+ path + ".options." + key,
301
+ "remove this field; the importer selects model/provider configuration from the buyer's account.",
302
+ )
303
+ # Older published files stay byte-for-byte intact. Their
304
+ # account-specific settings are never used or displayed.
305
+ options.pop(key)
306
+ if "position" in options:
307
+ string(options["position"], path + ".options.position")
308
+ if "params" in options:
309
+ params = obj(
310
+ options["params"],
311
+ NUMERIC_PARAMS
312
+ | BOOL_PARAMS
313
+ | {"stop", "stop_sequences", "reasoning_effort", "verbosity"},
314
+ set(),
315
+ path + ".options.params",
316
+ )
317
+ for key, value in params.items():
318
+ pp = path + ".options.params." + key
319
+ if key in NUMERIC_PARAMS and type(value) not in (int, float):
320
+ fail(pp, "must be a finite number.")
321
+ if key in BOOL_PARAMS and not isinstance(value, bool):
322
+ fail(pp, "must be boolean.")
323
+ if key in ("stop", "stop_sequences"):
324
+ strings(value, pp)
325
+ if key == "reasoning_effort" and value not in (
326
+ "none",
327
+ "minimal",
328
+ "low",
329
+ "medium",
330
+ "high",
331
+ ):
332
+ fail(pp, "use none, minimal, low, medium, or high.")
333
+ if key == "verbosity" and value not in ("low", "medium", "high"):
334
+ fail(pp, "use low, medium, or high.")
335
+ if kind == "task":
336
+ continue
337
+ parser = obj(
338
+ prompt["parser"],
339
+ {"type", "use_cot", "choice_scores", "allow_skip"},
340
+ {"type", "use_cot", "choice_scores"},
341
+ path + ".parser",
342
+ )
343
+ if parser["type"] != "llm_classifier":
344
+ fail(path + ".parser.type", "must be llm_classifier.")
345
+ for key in ("use_cot", "allow_skip"):
346
+ if key in parser and not isinstance(parser[key], bool):
347
+ fail(path + ".parser." + key, "must be boolean.")
348
+ choices = parser["choice_scores"]
349
+ if (
350
+ not isinstance(choices, dict)
351
+ or not choices
352
+ or any(
353
+ not key.strip()
354
+ or type(value) not in (int, float)
355
+ or not 0 <= value <= 1
356
+ for key, value in choices.items()
357
+ )
358
+ ):
359
+ fail(
360
+ path + ".parser.choice_scores",
361
+ "use a nonempty choice-to-score mapping with finite numbers from 0 to 1.",
362
+ )
363
+ else:
364
+ fail(
365
+ path + ".function_data.type",
366
+ "only code and prompt tasks/scorers are supported.",
367
+ )
368
+ return cases, tasks, scorers
evalmarket/cli.py ADDED
@@ -0,0 +1,103 @@
1
+ """Command-line parsing and user-facing import errors."""
2
+
3
+ import argparse
4
+ import subprocess
5
+ import sys
6
+ from pathlib import Path
7
+
8
+ from evalmarket._version import __version__
9
+ from evalmarket.braintrust.experiments import ExperimentStartError
10
+ from evalmarket.braintrust.importer import import_braintrust
11
+ from evalmarket.langsmith.importer import import_package
12
+ from evalmarket.langsmith.playground import PlaygroundSetupError
13
+
14
+ DESCRIPTION = (
15
+ "Import native eval files, then optionally start Braintrust or open LangSmith."
16
+ )
17
+
18
+
19
+ def add_import_arguments(parser):
20
+ parser.add_argument(
21
+ "folder",
22
+ help="Extracted platform folder: braintrust or langchain (or a path to it)",
23
+ )
24
+ parser.add_argument("--project-id", help="Braintrust destination project UUID")
25
+ parser.add_argument(
26
+ "--yes", action="store_true", help="Confirm the displayed Braintrust import"
27
+ )
28
+ parser.add_argument(
29
+ "--provider",
30
+ action="append",
31
+ help="Braintrust buyer provider selection: function-slug=discovered-provider-name (required when the model is ambiguous)",
32
+ )
33
+ parser.add_argument(
34
+ "--model",
35
+ action="append",
36
+ help="Braintrust buyer model selection: function-slug=discovered-model-name (required per prompt task/LLM scorer in unattended imports)",
37
+ )
38
+ parser.add_argument(
39
+ "--dataset",
40
+ help="Optional dataset name; defaults to the evaluator name plus a unique timestamp",
41
+ )
42
+ parser.add_argument(
43
+ "--profile",
44
+ help="Existing LangSmith CLI profile; otherwise use the CLI's current authentication",
45
+ )
46
+ parser.add_argument(
47
+ "--model-config",
48
+ help="Optional existing native model JSON; otherwise choose a saved LangSmith model interactively",
49
+ )
50
+
51
+
52
+ def main(argv=None):
53
+ parser = argparse.ArgumentParser(prog="evalmarket", description=DESCRIPTION)
54
+ parser.add_argument(
55
+ "--version", action="version", version=f"evalmarket {__version__}"
56
+ )
57
+ commands = parser.add_subparsers(dest="command", required=True)
58
+ importer = commands.add_parser(
59
+ "import", help="Import a native platform folder", description=DESCRIPTION
60
+ )
61
+ add_import_arguments(importer)
62
+ return run_import(parser.parse_args(argv))
63
+
64
+
65
+ def run_import(args):
66
+ try:
67
+ if Path(args.folder).name == "braintrust":
68
+ if args.dataset or args.profile or args.model_config:
69
+ raise ValueError(
70
+ "--dataset, --profile and --model-config are LangSmith-only options."
71
+ )
72
+ import_braintrust(
73
+ args.folder, args.project_id, args.yes, args.model, args.provider
74
+ )
75
+ else:
76
+ if args.project_id or args.yes or args.model or args.provider:
77
+ raise ValueError("Braintrust options require a braintrust/ folder.")
78
+ import_package(args.folder, args.dataset, args.profile, args.model_config)
79
+ except PlaygroundSetupError as error:
80
+ print(
81
+ f"\nImport complete; Playground setup needs attention.\n{error}",
82
+ file=sys.stderr,
83
+ )
84
+ return 1
85
+ except ExperimentStartError as error:
86
+ print(
87
+ f"\nImport complete; experiment start needs attention.\n{error}",
88
+ file=sys.stderr,
89
+ )
90
+ return 1
91
+ except (
92
+ OSError,
93
+ ValueError,
94
+ subprocess.CalledProcessError,
95
+ KeyboardInterrupt,
96
+ EOFError,
97
+ ) as error:
98
+ print(
99
+ f"Import stopped: {error}\nIf an upload started, inspect your destination workspace for partial resources before retrying; the CLI does not retry or replace existing resources.",
100
+ file=sys.stderr,
101
+ )
102
+ return 1
103
+ return 0