evalmarket-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalmarket/__init__.py +1 -0
- evalmarket/__main__.py +5 -0
- evalmarket/_version.py +3 -0
- evalmarket/braintrust/__init__.py +1 -0
- evalmarket/braintrust/client.py +153 -0
- evalmarket/braintrust/experiments.py +67 -0
- evalmarket/braintrust/importer.py +309 -0
- evalmarket/braintrust/models.py +169 -0
- evalmarket/braintrust/resources.py +21 -0
- evalmarket/braintrust/validation.py +368 -0
- evalmarket/cli.py +103 -0
- evalmarket/entrypoint.py +41 -0
- evalmarket/json_data.py +71 -0
- evalmarket/langsmith/__init__.py +1 -0
- evalmarket/langsmith/importer.py +246 -0
- evalmarket/langsmith/models.py +52 -0
- evalmarket/langsmith/playground.py +123 -0
- evalmarket/langsmith/prompts.py +72 -0
- evalmarket/langsmith/resources.py +12 -0
- evalmarket/langsmith/validation.py +86 -0
- evalmarket_cli-0.1.0.dist-info/METADATA +69 -0
- evalmarket_cli-0.1.0.dist-info/RECORD +24 -0
- evalmarket_cli-0.1.0.dist-info/WHEEL +4 -0
- evalmarket_cli-0.1.0.dist-info/entry_points.txt +2 -0
evalmarket/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""EvalMarket customer import tooling."""
|
evalmarket/__main__.py
ADDED
evalmarket/_version.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""EvalMarket braintrust."""
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Braintrust client for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from time import monotonic
|
|
5
|
+
from urllib.error import HTTPError, URLError
|
|
6
|
+
from urllib.parse import urlencode, urlsplit
|
|
7
|
+
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
|
8
|
+
from uuid import UUID
|
|
9
|
+
|
|
10
|
+
from evalmarket.braintrust.resources import model_label, resource_id
|
|
11
|
+
from evalmarket.json_data import native_json
|
|
12
|
+
|
|
13
|
+
BRAINTRUST_API = "https://api.braintrust.dev"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class NoCredentialRedirect(HTTPRedirectHandler):
|
|
17
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
18
|
+
raise ValueError("Braintrust redirect refused; credentials were not forwarded.")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BraintrustClient:
|
|
22
|
+
def __init__(self, key):
|
|
23
|
+
self.key = key
|
|
24
|
+
self.opener = build_opener(NoCredentialRedirect())
|
|
25
|
+
|
|
26
|
+
def request(
|
|
27
|
+
self, method, path, body=None, *, _start_only=False, _model_scope=None, **query
|
|
28
|
+
):
|
|
29
|
+
if not path.startswith("/v1/") or "?" in path or "#" in path:
|
|
30
|
+
raise ValueError("Unsupported Braintrust API path.")
|
|
31
|
+
headers = {
|
|
32
|
+
"Authorization": "Bearer " + self.key,
|
|
33
|
+
"Content-Type": "application/json",
|
|
34
|
+
"Accept": "text/event-stream" if _start_only else "application/json",
|
|
35
|
+
}
|
|
36
|
+
if _model_scope is not None:
|
|
37
|
+
project_id, provider_name = _model_scope
|
|
38
|
+
if (
|
|
39
|
+
method != "GET"
|
|
40
|
+
or path != "/v1/proxy/models"
|
|
41
|
+
or not model_label(provider_name)
|
|
42
|
+
):
|
|
43
|
+
raise ValueError("Invalid Braintrust model discovery scope.")
|
|
44
|
+
headers.update(
|
|
45
|
+
{
|
|
46
|
+
"x-bt-project-id": str(UUID(project_id)),
|
|
47
|
+
"x-bt-endpoint-name": provider_name,
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
url = BRAINTRUST_API + path + ("?" + urlencode(query) if query else "")
|
|
51
|
+
request = Request(
|
|
52
|
+
url,
|
|
53
|
+
method=method,
|
|
54
|
+
headers=headers,
|
|
55
|
+
data=json.dumps(body, allow_nan=False).encode()
|
|
56
|
+
if body is not None
|
|
57
|
+
else None,
|
|
58
|
+
)
|
|
59
|
+
try:
|
|
60
|
+
with self.opener.open(request, timeout=60) as response:
|
|
61
|
+
if _start_only:
|
|
62
|
+
return read_experiment_start(response, body["project_id"])
|
|
63
|
+
return native_json(response.read().decode("utf-8"))
|
|
64
|
+
except HTTPError as error:
|
|
65
|
+
# Never echo provider bodies/URLs; these can include private data or credentials.
|
|
66
|
+
detail = {
|
|
67
|
+
401: "invalid API key",
|
|
68
|
+
403: "insufficient permissions",
|
|
69
|
+
429: "rate limited",
|
|
70
|
+
}.get(error.code, "request rejected")
|
|
71
|
+
raise ValueError(
|
|
72
|
+
f"Braintrust HTTP {error.code}: {detail}. No write was retried."
|
|
73
|
+
) from None
|
|
74
|
+
except (URLError, OSError, UnicodeError, ValueError) as error:
|
|
75
|
+
if isinstance(error, ValueError) and str(error).startswith(
|
|
76
|
+
"Braintrust redirect"
|
|
77
|
+
):
|
|
78
|
+
raise
|
|
79
|
+
raise ValueError(
|
|
80
|
+
"Braintrust transport/read response failed; the outcome may be uncertain. No write was retried."
|
|
81
|
+
) from None
|
|
82
|
+
|
|
83
|
+
def objects(self, kind, **filters):
|
|
84
|
+
cursor, seen = None, set()
|
|
85
|
+
while True:
|
|
86
|
+
page = self.request(
|
|
87
|
+
"GET",
|
|
88
|
+
"/v1/" + kind,
|
|
89
|
+
**filters,
|
|
90
|
+
limit=100,
|
|
91
|
+
**({"starting_after": cursor} if cursor else {}),
|
|
92
|
+
)
|
|
93
|
+
records = page.get("objects") if isinstance(page, dict) else None
|
|
94
|
+
if not isinstance(records, list):
|
|
95
|
+
raise ValueError("Braintrust returned an invalid object list.")
|
|
96
|
+
if not records:
|
|
97
|
+
return
|
|
98
|
+
for record in records:
|
|
99
|
+
identity = resource_id(record)
|
|
100
|
+
if identity in seen:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"Braintrust pagination repeated an object; stopped safely."
|
|
103
|
+
)
|
|
104
|
+
seen.add(identity)
|
|
105
|
+
yield record
|
|
106
|
+
cursor = resource_id(records[-1])
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def read_experiment_start(response, project_id):
|
|
110
|
+
"""Read only the SSE start acknowledgement, never results or completion."""
|
|
111
|
+
event, data, size = "", [], 0
|
|
112
|
+
deadline = monotonic() + 60
|
|
113
|
+
while monotonic() < deadline:
|
|
114
|
+
line = response.readline(65537)
|
|
115
|
+
size += len(line)
|
|
116
|
+
if not line or size > 65536:
|
|
117
|
+
break
|
|
118
|
+
line = line.decode("utf-8").rstrip("\r\n")
|
|
119
|
+
if not line:
|
|
120
|
+
if event == "start" and data:
|
|
121
|
+
started = native_json("\n".join(data))
|
|
122
|
+
if (
|
|
123
|
+
not isinstance(started, dict)
|
|
124
|
+
or started.get("projectId") != project_id
|
|
125
|
+
):
|
|
126
|
+
break
|
|
127
|
+
identity = resource_id({"id": started.get("experimentId")})
|
|
128
|
+
url = started.get("experimentUrl")
|
|
129
|
+
if not isinstance(url, str) or any(
|
|
130
|
+
c.isspace() or ord(c) < 32 or ord(c) == 127 for c in url
|
|
131
|
+
):
|
|
132
|
+
break
|
|
133
|
+
parsed = urlsplit(url)
|
|
134
|
+
if (
|
|
135
|
+
parsed.scheme != "https"
|
|
136
|
+
or parsed.netloc != "www.braintrust.dev"
|
|
137
|
+
or not parsed.path.startswith("/app/")
|
|
138
|
+
or "/experiments/" not in parsed.path
|
|
139
|
+
):
|
|
140
|
+
break
|
|
141
|
+
return identity, url
|
|
142
|
+
if event or data:
|
|
143
|
+
# Do not parse error bodies or a result event as a successful start.
|
|
144
|
+
break
|
|
145
|
+
continue # SSE heartbeat/comment-only frame
|
|
146
|
+
field, separator, value = line.partition(":")
|
|
147
|
+
if separator and value.startswith(" "):
|
|
148
|
+
value = value[1:]
|
|
149
|
+
if field == "event":
|
|
150
|
+
event = value
|
|
151
|
+
elif field == "data":
|
|
152
|
+
data.append(value)
|
|
153
|
+
raise ValueError("No valid Braintrust experiment start acknowledgement.")
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Braintrust experiments for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
from uuid import uuid4
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ExperimentStartError(ValueError):
|
|
7
|
+
"""An optional run failed after a successful, verified import."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def offer_experiment(client, project_id, dataset_id, tasks, scorers, case_count):
|
|
11
|
+
print("\nStart an experiment")
|
|
12
|
+
print(
|
|
13
|
+
f" Runs one imported task on all {case_count} cases with the imported scorers."
|
|
14
|
+
)
|
|
15
|
+
print(
|
|
16
|
+
" Uses your Braintrust model/provider access; model calls may incur charges."
|
|
17
|
+
)
|
|
18
|
+
print(" Seller code runs with Braintrust hosted-function permissions.")
|
|
19
|
+
try:
|
|
20
|
+
answer = input("Would you like to start an experiment? [y/N]: ").strip().lower()
|
|
21
|
+
if answer not in {"y", "yes"}:
|
|
22
|
+
return False
|
|
23
|
+
selected = tasks[0]
|
|
24
|
+
if len(tasks) > 1:
|
|
25
|
+
choices = {str(i): task for i, task in enumerate(tasks, 1)}
|
|
26
|
+
while True:
|
|
27
|
+
print("\nChoose a task:")
|
|
28
|
+
for number, (task, _) in choices.items():
|
|
29
|
+
print(f" {number}. {task['name']!r}")
|
|
30
|
+
choice = input(f"Task number [1-{len(tasks)}, q to cancel]: ").strip()
|
|
31
|
+
if choice.lower() == "q":
|
|
32
|
+
return False
|
|
33
|
+
selected = choices.get(choice.lstrip("0"))
|
|
34
|
+
if selected is not None:
|
|
35
|
+
break
|
|
36
|
+
print(f"Enter a task number from 1 to {len(tasks)}, or q to cancel.")
|
|
37
|
+
except (EOFError, KeyboardInterrupt):
|
|
38
|
+
print()
|
|
39
|
+
return False
|
|
40
|
+
task, task_id = selected
|
|
41
|
+
name = "EvalMarket experiment " + uuid4().hex
|
|
42
|
+
print(f"\nStarting {name!r}", flush=True)
|
|
43
|
+
print(f" Task: {task['name']!r}", flush=True)
|
|
44
|
+
try:
|
|
45
|
+
identity, url = client.request(
|
|
46
|
+
"POST",
|
|
47
|
+
"/v1/eval",
|
|
48
|
+
{
|
|
49
|
+
"project_id": project_id,
|
|
50
|
+
"data": {"dataset_id": dataset_id},
|
|
51
|
+
"task": {"function_id": task_id},
|
|
52
|
+
"scores": [{"function_id": identity} for _, identity in scorers],
|
|
53
|
+
"experiment_name": name,
|
|
54
|
+
"stream": True,
|
|
55
|
+
"max_concurrency": 1,
|
|
56
|
+
"is_public": False,
|
|
57
|
+
},
|
|
58
|
+
_start_only=True,
|
|
59
|
+
)
|
|
60
|
+
except (ValueError, OSError, KeyboardInterrupt, EOFError):
|
|
61
|
+
raise ExperimentStartError(
|
|
62
|
+
f"Experiment start unconfirmed. Inspect project {project_id} for {name!r} before starting another run; it may already be running. No request was retried. Your imported resources are complete and do not need to be imported again."
|
|
63
|
+
) from None
|
|
64
|
+
print(f"\nExperiment started. ID: {identity}")
|
|
65
|
+
print(f" {url}")
|
|
66
|
+
print("Braintrust will continue the run. Open the link for progress and results.")
|
|
67
|
+
return True
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"""Braintrust importer for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import copy
|
|
4
|
+
import getpass
|
|
5
|
+
import os
|
|
6
|
+
import sys
|
|
7
|
+
import warnings
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from uuid import UUID, uuid4
|
|
10
|
+
|
|
11
|
+
from evalmarket.braintrust.client import BraintrustClient
|
|
12
|
+
from evalmarket.braintrust.experiments import offer_experiment
|
|
13
|
+
from evalmarket.braintrust.models import choose_braintrust_models
|
|
14
|
+
from evalmarket.braintrust.resources import model_label, resource_id
|
|
15
|
+
from evalmarket.braintrust.validation import braintrust_package
|
|
16
|
+
from evalmarket.json_data import MAX_NATIVE_FILE_BYTES, same_json, verify_fields
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def import_braintrust(folder, project_id=None, yes=False, models=None, providers=None):
|
|
20
|
+
folder = Path(folder).resolve()
|
|
21
|
+
|
|
22
|
+
def read(name):
|
|
23
|
+
path = folder / name
|
|
24
|
+
if (
|
|
25
|
+
path.is_symlink()
|
|
26
|
+
or not path.is_file()
|
|
27
|
+
or not 0 < path.stat().st_size <= MAX_NATIVE_FILE_BYTES
|
|
28
|
+
):
|
|
29
|
+
raise ValueError(
|
|
30
|
+
f"braintrust/{name}: required regular file must be nonempty and up to 5 MiB."
|
|
31
|
+
+ (
|
|
32
|
+
" Ask the seller for a complete updated package with task setup; older compatibility checks did not require tasks."
|
|
33
|
+
if name == "tasks.json"
|
|
34
|
+
else ""
|
|
35
|
+
)
|
|
36
|
+
)
|
|
37
|
+
return path.read_bytes()
|
|
38
|
+
|
|
39
|
+
cases, tasks, scorers = braintrust_package(read, discard_legacy_models=True)
|
|
40
|
+
functions = tasks + scorers
|
|
41
|
+
interactive = sys.stdin.isatty()
|
|
42
|
+
if not interactive and (not project_id or not yes):
|
|
43
|
+
raise ValueError(
|
|
44
|
+
"Noninteractive Braintrust imports require --project-id and --yes. Nothing was uploaded."
|
|
45
|
+
)
|
|
46
|
+
if project_id:
|
|
47
|
+
try:
|
|
48
|
+
project_id = str(UUID(project_id))
|
|
49
|
+
except ValueError:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
"--project-id must be a Braintrust project UUID."
|
|
52
|
+
) from None
|
|
53
|
+
for flag, supplied, field in (
|
|
54
|
+
("--model", models, "model"),
|
|
55
|
+
("--provider", providers, "endpoint_name"),
|
|
56
|
+
):
|
|
57
|
+
overrides = {}
|
|
58
|
+
for value in supplied or []:
|
|
59
|
+
slug, separator, label = value.partition("=")
|
|
60
|
+
if (
|
|
61
|
+
not separator
|
|
62
|
+
or not model_label(label)
|
|
63
|
+
or slug in overrides
|
|
64
|
+
or not any(
|
|
65
|
+
scorer["slug"] == slug
|
|
66
|
+
and scorer["function_data"]["type"] == "prompt"
|
|
67
|
+
for scorer in functions
|
|
68
|
+
)
|
|
69
|
+
):
|
|
70
|
+
raise ValueError(
|
|
71
|
+
f"{flag} requires a unique prompt task or LLM scorer slug=nonempty-name override."
|
|
72
|
+
)
|
|
73
|
+
overrides[slug] = label
|
|
74
|
+
for scorer in functions:
|
|
75
|
+
if scorer["slug"] in overrides:
|
|
76
|
+
scorer["prompt_data"]["options"][field] = overrides[scorer["slug"]]
|
|
77
|
+
key = os.environ.get("BRAINTRUST_API_KEY")
|
|
78
|
+
if not key:
|
|
79
|
+
if not interactive:
|
|
80
|
+
raise ValueError(
|
|
81
|
+
"Set BRAINTRUST_API_KEY locally, or use an interactive terminal for a masked prompt."
|
|
82
|
+
)
|
|
83
|
+
with warnings.catch_warnings():
|
|
84
|
+
warnings.simplefilter("error", getpass.GetPassWarning)
|
|
85
|
+
try:
|
|
86
|
+
key = getpass.getpass("Braintrust API key (hidden): ")
|
|
87
|
+
except (getpass.GetPassWarning, EOFError):
|
|
88
|
+
raise ValueError(
|
|
89
|
+
"A masked terminal prompt is unavailable. Set BRAINTRUST_API_KEY locally."
|
|
90
|
+
) from None
|
|
91
|
+
if not key or key.strip() != key or any(c.isspace() or ord(c) < 32 for c in key):
|
|
92
|
+
raise ValueError(
|
|
93
|
+
"A nonempty Braintrust API key without whitespace is required."
|
|
94
|
+
)
|
|
95
|
+
client = BraintrustClient(key)
|
|
96
|
+
if project_id:
|
|
97
|
+
project = client.request("GET", "/v1/project/" + project_id)
|
|
98
|
+
if resource_id(project) != project_id:
|
|
99
|
+
raise ValueError("The accessible project did not match --project-id.")
|
|
100
|
+
else:
|
|
101
|
+
projects = list(client.objects("project"))
|
|
102
|
+
if not projects:
|
|
103
|
+
raise ValueError(
|
|
104
|
+
"No accessible Braintrust projects. Create/configure one in Braintrust first."
|
|
105
|
+
)
|
|
106
|
+
project_numbers = {str(i): project for i, project in enumerate(projects, 1)}
|
|
107
|
+
while True:
|
|
108
|
+
print("\nAvailable Braintrust projects (choose the number on the left):")
|
|
109
|
+
for number, project in project_numbers.items():
|
|
110
|
+
print(
|
|
111
|
+
f" {number}. {project.get('name', 'Unnamed project')!r} ({resource_id(project)})"
|
|
112
|
+
)
|
|
113
|
+
try:
|
|
114
|
+
choice = input(
|
|
115
|
+
f"Destination project number [1-{len(projects)}, q to cancel]: "
|
|
116
|
+
).strip()
|
|
117
|
+
except EOFError:
|
|
118
|
+
raise ValueError(
|
|
119
|
+
"No project selected; no writes were made. Rerun in an interactive terminal or supply --project-id and --yes."
|
|
120
|
+
) from None
|
|
121
|
+
if choice.lower() == "q":
|
|
122
|
+
raise ValueError("Import cancelled; no writes were made.")
|
|
123
|
+
project = project_numbers.get(choice.lstrip("0"))
|
|
124
|
+
if project is not None:
|
|
125
|
+
break
|
|
126
|
+
print(f"Enter a project number from 1 to {len(projects)}, or q to cancel.")
|
|
127
|
+
project_id = resource_id(project)
|
|
128
|
+
choose_braintrust_models(client, project, functions, interactive)
|
|
129
|
+
token = uuid4().hex
|
|
130
|
+
name = "EvalMarket import " + token
|
|
131
|
+
print("\nBraintrust import")
|
|
132
|
+
print(f" Destination: {project.get('name', 'Unnamed project')!r} ({project_id})")
|
|
133
|
+
print(f" Dataset: {len(cases)} cases")
|
|
134
|
+
print(f" Import ID: {token}")
|
|
135
|
+
prepared = []
|
|
136
|
+
for kind, scorer in [("task", task) for task in tasks] + [
|
|
137
|
+
("scorer", scorer) for scorer in scorers
|
|
138
|
+
]:
|
|
139
|
+
native = copy.deepcopy(scorer)
|
|
140
|
+
native.update(
|
|
141
|
+
project_id=project_id,
|
|
142
|
+
name=scorer["name"] + " [" + token + "]",
|
|
143
|
+
slug=scorer["slug"] + "-" + token,
|
|
144
|
+
)
|
|
145
|
+
# The package role is "task", but Braintrust's Prompt picker lists
|
|
146
|
+
# answering prompts as "llm". "task" is callable via the API but hidden
|
|
147
|
+
# from that picker. Keep the seller's prompt and grading logic intact.
|
|
148
|
+
if kind == "task" and native["function_data"]["type"] == "prompt":
|
|
149
|
+
native["function_type"] = "llm"
|
|
150
|
+
prepared.append((kind, native))
|
|
151
|
+
print(f" {kind.title()}: {scorer['name']!r}")
|
|
152
|
+
if native["function_data"]["type"] == "prompt":
|
|
153
|
+
print(f" Model: {native['prompt_data']['options']['model']!r}")
|
|
154
|
+
print(
|
|
155
|
+
f" Provider: {native['prompt_data']['options'].get('endpoint_name', 'Braintrust routing default')!r}"
|
|
156
|
+
)
|
|
157
|
+
print(
|
|
158
|
+
"\nImport installs the evaluation without running it. Model changes can change outputs and judgments."
|
|
159
|
+
)
|
|
160
|
+
if (
|
|
161
|
+
not yes
|
|
162
|
+
and input("Create this dataset, task set, and scorer set? [y/N]: ")
|
|
163
|
+
.strip()
|
|
164
|
+
.lower()
|
|
165
|
+
!= "y"
|
|
166
|
+
):
|
|
167
|
+
raise ValueError("Import cancelled; no writes were made.")
|
|
168
|
+
# Check all identities before any write. No automatic retry or fallback name.
|
|
169
|
+
if list(client.objects("dataset", project_id=project_id, dataset_name=name)):
|
|
170
|
+
raise ValueError("Dataset name collision; no writes were made.")
|
|
171
|
+
for _, native in prepared:
|
|
172
|
+
if list(client.objects("function", project_id=project_id, slug=native["slug"])):
|
|
173
|
+
raise ValueError("Task/scorer slug collision; no writes were made.")
|
|
174
|
+
rows = []
|
|
175
|
+
row_ids = {case["id"] for case in cases if "id" in case}
|
|
176
|
+
for case in cases:
|
|
177
|
+
row = copy.deepcopy(case)
|
|
178
|
+
if "id" not in row:
|
|
179
|
+
identity = str(uuid4())
|
|
180
|
+
while identity in row_ids:
|
|
181
|
+
identity = str(uuid4())
|
|
182
|
+
row["id"] = identity
|
|
183
|
+
row_ids.add(identity)
|
|
184
|
+
rows.append(row)
|
|
185
|
+
confirmed, stage = [], "create dataset (outcome uncertain if no response)"
|
|
186
|
+
saved_tasks, saved_scorers = [], []
|
|
187
|
+
print("\nUploading and verifying...", flush=True)
|
|
188
|
+
try:
|
|
189
|
+
dataset_body = {
|
|
190
|
+
"project_id": project_id,
|
|
191
|
+
"name": name,
|
|
192
|
+
"description": "Seller-native EvalMarket import " + token,
|
|
193
|
+
"metadata": {"evalmarket_import_id": token},
|
|
194
|
+
}
|
|
195
|
+
dataset = client.request("POST", "/v1/dataset", dataset_body)
|
|
196
|
+
dataset_id = resource_id(dataset)
|
|
197
|
+
# The API reuses names. A foreign/missing marker must stop before insert.
|
|
198
|
+
verify_fields(dataset_body, dataset, "dataset")
|
|
199
|
+
confirmed.append("dataset created: " + dataset_id)
|
|
200
|
+
for start in range(0, len(rows), 100):
|
|
201
|
+
batch = rows[start : start + 100]
|
|
202
|
+
stage = f"upload dataset rows {start + 1}–{start + len(batch)} (outcome uncertain if interrupted)"
|
|
203
|
+
response = client.request(
|
|
204
|
+
"POST", f"/v1/dataset/{dataset_id}/insert", {"events": batch}
|
|
205
|
+
)
|
|
206
|
+
if not isinstance(response, dict) or response.get("row_ids") != [
|
|
207
|
+
row["id"] for row in batch
|
|
208
|
+
]:
|
|
209
|
+
raise ValueError(
|
|
210
|
+
"Dataset insert did not acknowledge the exact supplied row IDs."
|
|
211
|
+
)
|
|
212
|
+
confirmed.append(f"rows {start + 1}–{start + len(batch)} acknowledged")
|
|
213
|
+
stage = "dataset readback"
|
|
214
|
+
fetched, cursor, cursors = {}, None, set()
|
|
215
|
+
while True:
|
|
216
|
+
page = client.request(
|
|
217
|
+
"GET",
|
|
218
|
+
f"/v1/dataset/{dataset_id}/fetch",
|
|
219
|
+
limit=100,
|
|
220
|
+
**({"cursor": cursor} if cursor else {}),
|
|
221
|
+
)
|
|
222
|
+
if not isinstance(page, dict) or not isinstance(page.get("events"), list):
|
|
223
|
+
raise ValueError("Invalid dataset readback response.")
|
|
224
|
+
for row in page["events"]:
|
|
225
|
+
if not isinstance(row, dict) or not isinstance(row.get("id"), str):
|
|
226
|
+
raise ValueError("Invalid dataset readback row.")
|
|
227
|
+
# Fetch is newest-first; older versions can appear on later pages.
|
|
228
|
+
fetched.setdefault(row["id"], row)
|
|
229
|
+
cursor = page.get("cursor")
|
|
230
|
+
if not cursor:
|
|
231
|
+
break
|
|
232
|
+
if not isinstance(cursor, str) or cursor in cursors:
|
|
233
|
+
raise ValueError("Dataset readback cursor repeated or was invalid.")
|
|
234
|
+
cursors.add(cursor)
|
|
235
|
+
if fetched.keys() != row_ids:
|
|
236
|
+
raise ValueError("Dataset readback IDs differ from the uploaded case set.")
|
|
237
|
+
for row in rows:
|
|
238
|
+
actual = fetched[row["id"]]
|
|
239
|
+
for field in ("id", "input", "expected", "metadata", "tags"):
|
|
240
|
+
if field in row:
|
|
241
|
+
if field not in actual or not same_json(row[field], actual[field]):
|
|
242
|
+
raise ValueError(
|
|
243
|
+
f"Dataset readback mismatch in {field}; supplied values were not preserved."
|
|
244
|
+
)
|
|
245
|
+
elif actual.get(field) is not None:
|
|
246
|
+
raise ValueError(
|
|
247
|
+
f"Dataset readback introduced an unexpected {field} value."
|
|
248
|
+
)
|
|
249
|
+
confirmed.append("all dataset rows verified")
|
|
250
|
+
print(f" Dataset verified: {len(rows)} cases", flush=True)
|
|
251
|
+
for kind, native in prepared:
|
|
252
|
+
stage = (
|
|
253
|
+
"create "
|
|
254
|
+
+ kind
|
|
255
|
+
+ " "
|
|
256
|
+
+ native["slug"]
|
|
257
|
+
+ " (outcome uncertain if no response)"
|
|
258
|
+
)
|
|
259
|
+
created = client.request("POST", "/v1/function", native)
|
|
260
|
+
function_id = resource_id(created)
|
|
261
|
+
confirmed.append(
|
|
262
|
+
kind + " created: " + function_id + " (" + native["slug"] + ")"
|
|
263
|
+
)
|
|
264
|
+
stage = "readback " + kind + " " + function_id
|
|
265
|
+
verify_fields(
|
|
266
|
+
native, client.request("GET", "/v1/function/" + function_id), kind
|
|
267
|
+
)
|
|
268
|
+
confirmed.append(kind + " verified: " + function_id)
|
|
269
|
+
(saved_tasks if kind == "task" else saved_scorers).append(
|
|
270
|
+
(native, function_id)
|
|
271
|
+
)
|
|
272
|
+
print(f" {kind.title()} verified", flush=True)
|
|
273
|
+
except (ValueError, OSError, KeyboardInterrupt, EOFError):
|
|
274
|
+
print("\nImport incomplete. Stopped during: " + stage, file=sys.stderr)
|
|
275
|
+
print(
|
|
276
|
+
"Confirmed: " + ("; ".join(confirmed) or "no resources confirmed"),
|
|
277
|
+
file=sys.stderr,
|
|
278
|
+
)
|
|
279
|
+
print(
|
|
280
|
+
f"Inspect project {project_id}, dataset {name!r}, and functions with import suffix {token} before retrying. An interrupted write may have succeeded. Nothing was retried or deleted; a new invocation creates a new import.",
|
|
281
|
+
file=sys.stderr,
|
|
282
|
+
)
|
|
283
|
+
raise
|
|
284
|
+
print("\nImport complete. Dataset, tasks, and scorers are readback-verified.")
|
|
285
|
+
print(f" Dataset: {name!r}\n ID: {dataset_id}")
|
|
286
|
+
for kind, resources in (("Task", saved_tasks), ("Scorer", saved_scorers)):
|
|
287
|
+
for native, identity in resources:
|
|
288
|
+
print(f" {kind}: {native['name']!r}\n ID: {identity}")
|
|
289
|
+
print("Tasks receive case input; expected answers stay with scorers.")
|
|
290
|
+
if interactive and offer_experiment(
|
|
291
|
+
client, project_id, dataset_id, saved_tasks, saved_scorers, len(cases)
|
|
292
|
+
):
|
|
293
|
+
return dataset_id
|
|
294
|
+
print("\nNo evaluation was run. Your imported resources are ready.")
|
|
295
|
+
print(
|
|
296
|
+
f"Open https://www.braintrust.dev and project {project.get('name', project_id)!r}."
|
|
297
|
+
)
|
|
298
|
+
if any(task["function_data"]["type"] == "prompt" for task in tasks):
|
|
299
|
+
print(
|
|
300
|
+
" Experiments > + Experiment > + Task > Prompt: choose the imported prompt, dataset and scorers. Refresh if needed."
|
|
301
|
+
)
|
|
302
|
+
if any(task["function_data"]["type"] == "code" for task in tasks):
|
|
303
|
+
print(
|
|
304
|
+
" Inline code tasks: use their saved IDs with the API/SDK. The UI task picker does not list them."
|
|
305
|
+
)
|
|
306
|
+
print(
|
|
307
|
+
"Configure model/provider access before running; missing access is a setup failure. Seller code runs with Braintrust hosted-function permissions."
|
|
308
|
+
)
|
|
309
|
+
return dataset_id
|