evalmarket-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
evalmarket/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """EvalMarket customer import tooling."""
evalmarket/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ """Support `python -I -m evalmarket` with the same CLI entry point."""
2
+
3
+ from evalmarket.entrypoint import main
4
+
5
+ raise SystemExit(main())
evalmarket/_version.py ADDED
@@ -0,0 +1,3 @@
1
+ """Version used by the CLI and distribution metadata."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1 @@
1
+ """EvalMarket braintrust."""
@@ -0,0 +1,153 @@
1
+ """Braintrust client for native evaluation imports."""
2
+
3
+ import json
4
+ from time import monotonic
5
+ from urllib.error import HTTPError, URLError
6
+ from urllib.parse import urlencode, urlsplit
7
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
8
+ from uuid import UUID
9
+
10
+ from evalmarket.braintrust.resources import model_label, resource_id
11
+ from evalmarket.json_data import native_json
12
+
13
+ BRAINTRUST_API = "https://api.braintrust.dev"
14
+
15
+
16
+ class NoCredentialRedirect(HTTPRedirectHandler):
17
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
18
+ raise ValueError("Braintrust redirect refused; credentials were not forwarded.")
19
+
20
+
21
+ class BraintrustClient:
22
+ def __init__(self, key):
23
+ self.key = key
24
+ self.opener = build_opener(NoCredentialRedirect())
25
+
26
+ def request(
27
+ self, method, path, body=None, *, _start_only=False, _model_scope=None, **query
28
+ ):
29
+ if not path.startswith("/v1/") or "?" in path or "#" in path:
30
+ raise ValueError("Unsupported Braintrust API path.")
31
+ headers = {
32
+ "Authorization": "Bearer " + self.key,
33
+ "Content-Type": "application/json",
34
+ "Accept": "text/event-stream" if _start_only else "application/json",
35
+ }
36
+ if _model_scope is not None:
37
+ project_id, provider_name = _model_scope
38
+ if (
39
+ method != "GET"
40
+ or path != "/v1/proxy/models"
41
+ or not model_label(provider_name)
42
+ ):
43
+ raise ValueError("Invalid Braintrust model discovery scope.")
44
+ headers.update(
45
+ {
46
+ "x-bt-project-id": str(UUID(project_id)),
47
+ "x-bt-endpoint-name": provider_name,
48
+ }
49
+ )
50
+ url = BRAINTRUST_API + path + ("?" + urlencode(query) if query else "")
51
+ request = Request(
52
+ url,
53
+ method=method,
54
+ headers=headers,
55
+ data=json.dumps(body, allow_nan=False).encode()
56
+ if body is not None
57
+ else None,
58
+ )
59
+ try:
60
+ with self.opener.open(request, timeout=60) as response:
61
+ if _start_only:
62
+ return read_experiment_start(response, body["project_id"])
63
+ return native_json(response.read().decode("utf-8"))
64
+ except HTTPError as error:
65
+ # Never echo provider bodies/URLs; these can include private data or credentials.
66
+ detail = {
67
+ 401: "invalid API key",
68
+ 403: "insufficient permissions",
69
+ 429: "rate limited",
70
+ }.get(error.code, "request rejected")
71
+ raise ValueError(
72
+ f"Braintrust HTTP {error.code}: {detail}. No write was retried."
73
+ ) from None
74
+ except (URLError, OSError, UnicodeError, ValueError) as error:
75
+ if isinstance(error, ValueError) and str(error).startswith(
76
+ "Braintrust redirect"
77
+ ):
78
+ raise
79
+ raise ValueError(
80
+ "Braintrust transport/read response failed; the outcome may be uncertain. No write was retried."
81
+ ) from None
82
+
83
+ def objects(self, kind, **filters):
84
+ cursor, seen = None, set()
85
+ while True:
86
+ page = self.request(
87
+ "GET",
88
+ "/v1/" + kind,
89
+ **filters,
90
+ limit=100,
91
+ **({"starting_after": cursor} if cursor else {}),
92
+ )
93
+ records = page.get("objects") if isinstance(page, dict) else None
94
+ if not isinstance(records, list):
95
+ raise ValueError("Braintrust returned an invalid object list.")
96
+ if not records:
97
+ return
98
+ for record in records:
99
+ identity = resource_id(record)
100
+ if identity in seen:
101
+ raise ValueError(
102
+ "Braintrust pagination repeated an object; stopped safely."
103
+ )
104
+ seen.add(identity)
105
+ yield record
106
+ cursor = resource_id(records[-1])
107
+
108
+
109
+ def read_experiment_start(response, project_id):
110
+ """Read only the SSE start acknowledgement, never results or completion."""
111
+ event, data, size = "", [], 0
112
+ deadline = monotonic() + 60
113
+ while monotonic() < deadline:
114
+ line = response.readline(65537)
115
+ size += len(line)
116
+ if not line or size > 65536:
117
+ break
118
+ line = line.decode("utf-8").rstrip("\r\n")
119
+ if not line:
120
+ if event == "start" and data:
121
+ started = native_json("\n".join(data))
122
+ if (
123
+ not isinstance(started, dict)
124
+ or started.get("projectId") != project_id
125
+ ):
126
+ break
127
+ identity = resource_id({"id": started.get("experimentId")})
128
+ url = started.get("experimentUrl")
129
+ if not isinstance(url, str) or any(
130
+ c.isspace() or ord(c) < 32 or ord(c) == 127 for c in url
131
+ ):
132
+ break
133
+ parsed = urlsplit(url)
134
+ if (
135
+ parsed.scheme != "https"
136
+ or parsed.netloc != "www.braintrust.dev"
137
+ or not parsed.path.startswith("/app/")
138
+ or "/experiments/" not in parsed.path
139
+ ):
140
+ break
141
+ return identity, url
142
+ if event or data:
143
+ # Do not parse error bodies or a result event as a successful start.
144
+ break
145
+ continue # SSE heartbeat/comment-only frame
146
+ field, separator, value = line.partition(":")
147
+ if separator and value.startswith(" "):
148
+ value = value[1:]
149
+ if field == "event":
150
+ event = value
151
+ elif field == "data":
152
+ data.append(value)
153
+ raise ValueError("No valid Braintrust experiment start acknowledgement.")
@@ -0,0 +1,67 @@
1
+ """Braintrust experiments for native evaluation imports."""
2
+
3
+ from uuid import uuid4
4
+
5
+
6
+ class ExperimentStartError(ValueError):
7
+ """An optional run failed after a successful, verified import."""
8
+
9
+
10
+ def offer_experiment(client, project_id, dataset_id, tasks, scorers, case_count):
11
+ print("\nStart an experiment")
12
+ print(
13
+ f" Runs one imported task on all {case_count} cases with the imported scorers."
14
+ )
15
+ print(
16
+ " Uses your Braintrust model/provider access; model calls may incur charges."
17
+ )
18
+ print(" Seller code runs with Braintrust hosted-function permissions.")
19
+ try:
20
+ answer = input("Would you like to start an experiment? [y/N]: ").strip().lower()
21
+ if answer not in {"y", "yes"}:
22
+ return False
23
+ selected = tasks[0]
24
+ if len(tasks) > 1:
25
+ choices = {str(i): task for i, task in enumerate(tasks, 1)}
26
+ while True:
27
+ print("\nChoose a task:")
28
+ for number, (task, _) in choices.items():
29
+ print(f" {number}. {task['name']!r}")
30
+ choice = input(f"Task number [1-{len(tasks)}, q to cancel]: ").strip()
31
+ if choice.lower() == "q":
32
+ return False
33
+ selected = choices.get(choice.lstrip("0"))
34
+ if selected is not None:
35
+ break
36
+ print(f"Enter a task number from 1 to {len(tasks)}, or q to cancel.")
37
+ except (EOFError, KeyboardInterrupt):
38
+ print()
39
+ return False
40
+ task, task_id = selected
41
+ name = "EvalMarket experiment " + uuid4().hex
42
+ print(f"\nStarting {name!r}", flush=True)
43
+ print(f" Task: {task['name']!r}", flush=True)
44
+ try:
45
+ identity, url = client.request(
46
+ "POST",
47
+ "/v1/eval",
48
+ {
49
+ "project_id": project_id,
50
+ "data": {"dataset_id": dataset_id},
51
+ "task": {"function_id": task_id},
52
+ "scores": [{"function_id": identity} for _, identity in scorers],
53
+ "experiment_name": name,
54
+ "stream": True,
55
+ "max_concurrency": 1,
56
+ "is_public": False,
57
+ },
58
+ _start_only=True,
59
+ )
60
+ except (ValueError, OSError, KeyboardInterrupt, EOFError):
61
+ raise ExperimentStartError(
62
+ f"Experiment start unconfirmed. Inspect project {project_id} for {name!r} before starting another run; it may already be running. No request was retried. Your imported resources are complete and do not need to be imported again."
63
+ ) from None
64
+ print(f"\nExperiment started. ID: {identity}")
65
+ print(f" {url}")
66
+ print("Braintrust will continue the run. Open the link for progress and results.")
67
+ return True
@@ -0,0 +1,309 @@
1
+ """Braintrust importer for native evaluation imports."""
2
+
3
+ import copy
4
+ import getpass
5
+ import os
6
+ import sys
7
+ import warnings
8
+ from pathlib import Path
9
+ from uuid import UUID, uuid4
10
+
11
+ from evalmarket.braintrust.client import BraintrustClient
12
+ from evalmarket.braintrust.experiments import offer_experiment
13
+ from evalmarket.braintrust.models import choose_braintrust_models
14
+ from evalmarket.braintrust.resources import model_label, resource_id
15
+ from evalmarket.braintrust.validation import braintrust_package
16
+ from evalmarket.json_data import MAX_NATIVE_FILE_BYTES, same_json, verify_fields
17
+
18
+
19
+ def import_braintrust(folder, project_id=None, yes=False, models=None, providers=None):
20
+ folder = Path(folder).resolve()
21
+
22
+ def read(name):
23
+ path = folder / name
24
+ if (
25
+ path.is_symlink()
26
+ or not path.is_file()
27
+ or not 0 < path.stat().st_size <= MAX_NATIVE_FILE_BYTES
28
+ ):
29
+ raise ValueError(
30
+ f"braintrust/{name}: required regular file must be nonempty and up to 5 MiB."
31
+ + (
32
+ " Ask the seller for a complete updated package with task setup; older compatibility checks did not require tasks."
33
+ if name == "tasks.json"
34
+ else ""
35
+ )
36
+ )
37
+ return path.read_bytes()
38
+
39
+ cases, tasks, scorers = braintrust_package(read, discard_legacy_models=True)
40
+ functions = tasks + scorers
41
+ interactive = sys.stdin.isatty()
42
+ if not interactive and (not project_id or not yes):
43
+ raise ValueError(
44
+ "Noninteractive Braintrust imports require --project-id and --yes. Nothing was uploaded."
45
+ )
46
+ if project_id:
47
+ try:
48
+ project_id = str(UUID(project_id))
49
+ except ValueError:
50
+ raise ValueError(
51
+ "--project-id must be a Braintrust project UUID."
52
+ ) from None
53
+ for flag, supplied, field in (
54
+ ("--model", models, "model"),
55
+ ("--provider", providers, "endpoint_name"),
56
+ ):
57
+ overrides = {}
58
+ for value in supplied or []:
59
+ slug, separator, label = value.partition("=")
60
+ if (
61
+ not separator
62
+ or not model_label(label)
63
+ or slug in overrides
64
+ or not any(
65
+ scorer["slug"] == slug
66
+ and scorer["function_data"]["type"] == "prompt"
67
+ for scorer in functions
68
+ )
69
+ ):
70
+ raise ValueError(
71
+ f"{flag} requires a unique prompt task or LLM scorer slug=nonempty-name override."
72
+ )
73
+ overrides[slug] = label
74
+ for scorer in functions:
75
+ if scorer["slug"] in overrides:
76
+ scorer["prompt_data"]["options"][field] = overrides[scorer["slug"]]
77
+ key = os.environ.get("BRAINTRUST_API_KEY")
78
+ if not key:
79
+ if not interactive:
80
+ raise ValueError(
81
+ "Set BRAINTRUST_API_KEY locally, or use an interactive terminal for a masked prompt."
82
+ )
83
+ with warnings.catch_warnings():
84
+ warnings.simplefilter("error", getpass.GetPassWarning)
85
+ try:
86
+ key = getpass.getpass("Braintrust API key (hidden): ")
87
+ except (getpass.GetPassWarning, EOFError):
88
+ raise ValueError(
89
+ "A masked terminal prompt is unavailable. Set BRAINTRUST_API_KEY locally."
90
+ ) from None
91
+ if not key or key.strip() != key or any(c.isspace() or ord(c) < 32 for c in key):
92
+ raise ValueError(
93
+ "A nonempty Braintrust API key without whitespace is required."
94
+ )
95
+ client = BraintrustClient(key)
96
+ if project_id:
97
+ project = client.request("GET", "/v1/project/" + project_id)
98
+ if resource_id(project) != project_id:
99
+ raise ValueError("The accessible project did not match --project-id.")
100
+ else:
101
+ projects = list(client.objects("project"))
102
+ if not projects:
103
+ raise ValueError(
104
+ "No accessible Braintrust projects. Create/configure one in Braintrust first."
105
+ )
106
+ project_numbers = {str(i): project for i, project in enumerate(projects, 1)}
107
+ while True:
108
+ print("\nAvailable Braintrust projects (choose the number on the left):")
109
+ for number, project in project_numbers.items():
110
+ print(
111
+ f" {number}. {project.get('name', 'Unnamed project')!r} ({resource_id(project)})"
112
+ )
113
+ try:
114
+ choice = input(
115
+ f"Destination project number [1-{len(projects)}, q to cancel]: "
116
+ ).strip()
117
+ except EOFError:
118
+ raise ValueError(
119
+ "No project selected; no writes were made. Rerun in an interactive terminal or supply --project-id and --yes."
120
+ ) from None
121
+ if choice.lower() == "q":
122
+ raise ValueError("Import cancelled; no writes were made.")
123
+ project = project_numbers.get(choice.lstrip("0"))
124
+ if project is not None:
125
+ break
126
+ print(f"Enter a project number from 1 to {len(projects)}, or q to cancel.")
127
+ project_id = resource_id(project)
128
+ choose_braintrust_models(client, project, functions, interactive)
129
+ token = uuid4().hex
130
+ name = "EvalMarket import " + token
131
+ print("\nBraintrust import")
132
+ print(f" Destination: {project.get('name', 'Unnamed project')!r} ({project_id})")
133
+ print(f" Dataset: {len(cases)} cases")
134
+ print(f" Import ID: {token}")
135
+ prepared = []
136
+ for kind, scorer in [("task", task) for task in tasks] + [
137
+ ("scorer", scorer) for scorer in scorers
138
+ ]:
139
+ native = copy.deepcopy(scorer)
140
+ native.update(
141
+ project_id=project_id,
142
+ name=scorer["name"] + " [" + token + "]",
143
+ slug=scorer["slug"] + "-" + token,
144
+ )
145
+ # The package role is "task", but Braintrust's Prompt picker lists
146
+ # answering prompts as "llm". "task" is callable via the API but hidden
147
+ # from that picker. Keep the seller's prompt and grading logic intact.
148
+ if kind == "task" and native["function_data"]["type"] == "prompt":
149
+ native["function_type"] = "llm"
150
+ prepared.append((kind, native))
151
+ print(f" {kind.title()}: {scorer['name']!r}")
152
+ if native["function_data"]["type"] == "prompt":
153
+ print(f" Model: {native['prompt_data']['options']['model']!r}")
154
+ print(
155
+ f" Provider: {native['prompt_data']['options'].get('endpoint_name', 'Braintrust routing default')!r}"
156
+ )
157
+ print(
158
+ "\nImport installs the evaluation without running it. Model changes can change outputs and judgments."
159
+ )
160
+ if (
161
+ not yes
162
+ and input("Create this dataset, task set, and scorer set? [y/N]: ")
163
+ .strip()
164
+ .lower()
165
+ != "y"
166
+ ):
167
+ raise ValueError("Import cancelled; no writes were made.")
168
+ # Check all identities before any write. No automatic retry or fallback name.
169
+ if list(client.objects("dataset", project_id=project_id, dataset_name=name)):
170
+ raise ValueError("Dataset name collision; no writes were made.")
171
+ for _, native in prepared:
172
+ if list(client.objects("function", project_id=project_id, slug=native["slug"])):
173
+ raise ValueError("Task/scorer slug collision; no writes were made.")
174
+ rows = []
175
+ row_ids = {case["id"] for case in cases if "id" in case}
176
+ for case in cases:
177
+ row = copy.deepcopy(case)
178
+ if "id" not in row:
179
+ identity = str(uuid4())
180
+ while identity in row_ids:
181
+ identity = str(uuid4())
182
+ row["id"] = identity
183
+ row_ids.add(identity)
184
+ rows.append(row)
185
+ confirmed, stage = [], "create dataset (outcome uncertain if no response)"
186
+ saved_tasks, saved_scorers = [], []
187
+ print("\nUploading and verifying...", flush=True)
188
+ try:
189
+ dataset_body = {
190
+ "project_id": project_id,
191
+ "name": name,
192
+ "description": "Seller-native EvalMarket import " + token,
193
+ "metadata": {"evalmarket_import_id": token},
194
+ }
195
+ dataset = client.request("POST", "/v1/dataset", dataset_body)
196
+ dataset_id = resource_id(dataset)
197
+ # The API reuses names. A foreign/missing marker must stop before insert.
198
+ verify_fields(dataset_body, dataset, "dataset")
199
+ confirmed.append("dataset created: " + dataset_id)
200
+ for start in range(0, len(rows), 100):
201
+ batch = rows[start : start + 100]
202
+ stage = f"upload dataset rows {start + 1}–{start + len(batch)} (outcome uncertain if interrupted)"
203
+ response = client.request(
204
+ "POST", f"/v1/dataset/{dataset_id}/insert", {"events": batch}
205
+ )
206
+ if not isinstance(response, dict) or response.get("row_ids") != [
207
+ row["id"] for row in batch
208
+ ]:
209
+ raise ValueError(
210
+ "Dataset insert did not acknowledge the exact supplied row IDs."
211
+ )
212
+ confirmed.append(f"rows {start + 1}–{start + len(batch)} acknowledged")
213
+ stage = "dataset readback"
214
+ fetched, cursor, cursors = {}, None, set()
215
+ while True:
216
+ page = client.request(
217
+ "GET",
218
+ f"/v1/dataset/{dataset_id}/fetch",
219
+ limit=100,
220
+ **({"cursor": cursor} if cursor else {}),
221
+ )
222
+ if not isinstance(page, dict) or not isinstance(page.get("events"), list):
223
+ raise ValueError("Invalid dataset readback response.")
224
+ for row in page["events"]:
225
+ if not isinstance(row, dict) or not isinstance(row.get("id"), str):
226
+ raise ValueError("Invalid dataset readback row.")
227
+ # Fetch is newest-first; older versions can appear on later pages.
228
+ fetched.setdefault(row["id"], row)
229
+ cursor = page.get("cursor")
230
+ if not cursor:
231
+ break
232
+ if not isinstance(cursor, str) or cursor in cursors:
233
+ raise ValueError("Dataset readback cursor repeated or was invalid.")
234
+ cursors.add(cursor)
235
+ if fetched.keys() != row_ids:
236
+ raise ValueError("Dataset readback IDs differ from the uploaded case set.")
237
+ for row in rows:
238
+ actual = fetched[row["id"]]
239
+ for field in ("id", "input", "expected", "metadata", "tags"):
240
+ if field in row:
241
+ if field not in actual or not same_json(row[field], actual[field]):
242
+ raise ValueError(
243
+ f"Dataset readback mismatch in {field}; supplied values were not preserved."
244
+ )
245
+ elif actual.get(field) is not None:
246
+ raise ValueError(
247
+ f"Dataset readback introduced an unexpected {field} value."
248
+ )
249
+ confirmed.append("all dataset rows verified")
250
+ print(f" Dataset verified: {len(rows)} cases", flush=True)
251
+ for kind, native in prepared:
252
+ stage = (
253
+ "create "
254
+ + kind
255
+ + " "
256
+ + native["slug"]
257
+ + " (outcome uncertain if no response)"
258
+ )
259
+ created = client.request("POST", "/v1/function", native)
260
+ function_id = resource_id(created)
261
+ confirmed.append(
262
+ kind + " created: " + function_id + " (" + native["slug"] + ")"
263
+ )
264
+ stage = "readback " + kind + " " + function_id
265
+ verify_fields(
266
+ native, client.request("GET", "/v1/function/" + function_id), kind
267
+ )
268
+ confirmed.append(kind + " verified: " + function_id)
269
+ (saved_tasks if kind == "task" else saved_scorers).append(
270
+ (native, function_id)
271
+ )
272
+ print(f" {kind.title()} verified", flush=True)
273
+ except (ValueError, OSError, KeyboardInterrupt, EOFError):
274
+ print("\nImport incomplete. Stopped during: " + stage, file=sys.stderr)
275
+ print(
276
+ "Confirmed: " + ("; ".join(confirmed) or "no resources confirmed"),
277
+ file=sys.stderr,
278
+ )
279
+ print(
280
+ f"Inspect project {project_id}, dataset {name!r}, and functions with import suffix {token} before retrying. An interrupted write may have succeeded. Nothing was retried or deleted; a new invocation creates a new import.",
281
+ file=sys.stderr,
282
+ )
283
+ raise
284
+ print("\nImport complete. Dataset, tasks, and scorers are readback-verified.")
285
+ print(f" Dataset: {name!r}\n ID: {dataset_id}")
286
+ for kind, resources in (("Task", saved_tasks), ("Scorer", saved_scorers)):
287
+ for native, identity in resources:
288
+ print(f" {kind}: {native['name']!r}\n ID: {identity}")
289
+ print("Tasks receive case input; expected answers stay with scorers.")
290
+ if interactive and offer_experiment(
291
+ client, project_id, dataset_id, saved_tasks, saved_scorers, len(cases)
292
+ ):
293
+ return dataset_id
294
+ print("\nNo evaluation was run. Your imported resources are ready.")
295
+ print(
296
+ f"Open https://www.braintrust.dev and project {project.get('name', project_id)!r}."
297
+ )
298
+ if any(task["function_data"]["type"] == "prompt" for task in tasks):
299
+ print(
300
+ " Experiments > + Experiment > + Task > Prompt: choose the imported prompt, dataset and scorers. Refresh if needed."
301
+ )
302
+ if any(task["function_data"]["type"] == "code" for task in tasks):
303
+ print(
304
+ " Inline code tasks: use their saved IDs with the API/SDK. The UI task picker does not list them."
305
+ )
306
+ print(
307
+ "Configure model/provider access before running; missing access is a setup failure. Seller code runs with Braintrust hosted-function permissions."
308
+ )
309
+ return dataset_id