evalmarket-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalmarket/__init__.py +1 -0
- evalmarket/__main__.py +5 -0
- evalmarket/_version.py +3 -0
- evalmarket/braintrust/__init__.py +1 -0
- evalmarket/braintrust/client.py +153 -0
- evalmarket/braintrust/experiments.py +67 -0
- evalmarket/braintrust/importer.py +309 -0
- evalmarket/braintrust/models.py +169 -0
- evalmarket/braintrust/resources.py +21 -0
- evalmarket/braintrust/validation.py +368 -0
- evalmarket/cli.py +103 -0
- evalmarket/entrypoint.py +41 -0
- evalmarket/json_data.py +71 -0
- evalmarket/langsmith/__init__.py +1 -0
- evalmarket/langsmith/importer.py +246 -0
- evalmarket/langsmith/models.py +52 -0
- evalmarket/langsmith/playground.py +123 -0
- evalmarket/langsmith/prompts.py +72 -0
- evalmarket/langsmith/resources.py +12 -0
- evalmarket/langsmith/validation.py +86 -0
- evalmarket_cli-0.1.0.dist-info/METADATA +69 -0
- evalmarket_cli-0.1.0.dist-info/RECORD +24 -0
- evalmarket_cli-0.1.0.dist-info/WHEEL +4 -0
- evalmarket_cli-0.1.0.dist-info/entry_points.txt +2 -0
evalmarket/entrypoint.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Isolate CLI startup before importing file-based modules or seller data."""
|
|
2
|
+
|
|
3
|
+
import sys
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def restart_isolated(arguments):
|
|
7
|
+
"""Restart with built-ins only, preserving arguments, environment and status."""
|
|
8
|
+
command = [sys.executable, "-I", *arguments]
|
|
9
|
+
if sys.platform == "win32":
|
|
10
|
+
# Windows execv loses the child's exit status. spawnv waits, but needs
|
|
11
|
+
# Windows CRT quoting. Do not import subprocess before isolation.
|
|
12
|
+
def quote_windows_argument(value):
|
|
13
|
+
quoted = '"'
|
|
14
|
+
backslashes = 0
|
|
15
|
+
for char in value:
|
|
16
|
+
if char == "\\":
|
|
17
|
+
backslashes += 1
|
|
18
|
+
continue
|
|
19
|
+
quoted += "\\" * (backslashes * 2 + 1 if char == '"' else backslashes)
|
|
20
|
+
quoted += char
|
|
21
|
+
backslashes = 0
|
|
22
|
+
return quoted + "\\" * (backslashes * 2) + '"'
|
|
23
|
+
|
|
24
|
+
native = __import__("nt")
|
|
25
|
+
raise SystemExit(
|
|
26
|
+
native.spawnv(
|
|
27
|
+
native.P_WAIT,
|
|
28
|
+
sys.executable,
|
|
29
|
+
[quote_windows_argument(arg) for arg in command],
|
|
30
|
+
)
|
|
31
|
+
)
|
|
32
|
+
__import__("posix").execv(sys.executable, command)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def main():
|
|
36
|
+
if not sys.flags.isolated:
|
|
37
|
+
restart_isolated(["-m", "evalmarket", *sys.argv[1:]])
|
|
38
|
+
|
|
39
|
+
from evalmarket.cli import main as run
|
|
40
|
+
|
|
41
|
+
return run()
|
evalmarket/json_data.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Json_data for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
|
|
6
|
+
MAX_NATIVE_FILE_BYTES = 5 * 1024 * 1024
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def native_json(text):
|
|
10
|
+
"""Reject duplicate keys and non-finite values, including exponent overflow."""
|
|
11
|
+
|
|
12
|
+
def pairs(items):
|
|
13
|
+
result = {}
|
|
14
|
+
for key, value in items:
|
|
15
|
+
if key in result:
|
|
16
|
+
raise ValueError("Duplicate JSON object key.")
|
|
17
|
+
result[key] = value
|
|
18
|
+
return result
|
|
19
|
+
|
|
20
|
+
value = json.loads(text, object_pairs_hook=pairs)
|
|
21
|
+
|
|
22
|
+
def finite(item):
|
|
23
|
+
if isinstance(item, float) and not math.isfinite(item):
|
|
24
|
+
raise ValueError("JSON numbers must be finite.")
|
|
25
|
+
if isinstance(item, dict):
|
|
26
|
+
for child in item.values():
|
|
27
|
+
finite(child)
|
|
28
|
+
elif isinstance(item, list):
|
|
29
|
+
for child in item:
|
|
30
|
+
finite(child)
|
|
31
|
+
|
|
32
|
+
finite(value)
|
|
33
|
+
return value
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def same_json(left, right):
|
|
37
|
+
"""JSON-aware equality distinguishes booleans from numbers at every depth."""
|
|
38
|
+
if isinstance(left, dict):
|
|
39
|
+
return (
|
|
40
|
+
isinstance(right, dict)
|
|
41
|
+
and left.keys() == right.keys()
|
|
42
|
+
and all(same_json(v, right[k]) for k, v in left.items())
|
|
43
|
+
)
|
|
44
|
+
if isinstance(left, list):
|
|
45
|
+
return (
|
|
46
|
+
isinstance(right, list)
|
|
47
|
+
and len(left) == len(right)
|
|
48
|
+
and all(same_json(a, b) for a, b in zip(left, right))
|
|
49
|
+
)
|
|
50
|
+
if type(left) in (int, float):
|
|
51
|
+
return type(right) in (int, float) and left == right
|
|
52
|
+
return type(left) is type(right) and left == right
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def verify_fields(expected, actual, path="resource"):
|
|
56
|
+
"""Supplied values must survive; provider-added optional null fields may appear."""
|
|
57
|
+
if isinstance(expected, dict) and isinstance(actual, dict):
|
|
58
|
+
if path not in {"resource", "dataset", "task", "scorer"}:
|
|
59
|
+
for key in actual.keys() - expected.keys():
|
|
60
|
+
if actual[key] is not None and not (
|
|
61
|
+
path.endswith(".function_data.data") and key == "code_hash"
|
|
62
|
+
):
|
|
63
|
+
raise ValueError(
|
|
64
|
+
f"Readback mismatch at {path}; unexpected non-null fields."
|
|
65
|
+
)
|
|
66
|
+
for key, value in expected.items():
|
|
67
|
+
if key not in actual:
|
|
68
|
+
raise ValueError(f"Readback mismatch at {path}.{key} (missing field).")
|
|
69
|
+
verify_fields(value, actual[key], path + "." + key)
|
|
70
|
+
elif not same_json(expected, actual):
|
|
71
|
+
raise ValueError(f"Readback mismatch at {path}; import is not verified.")
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""EvalMarket langsmith."""
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"""Langsmith importer for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import shutil
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from evalmarket.json_data import MAX_NATIVE_FILE_BYTES, native_json
|
|
11
|
+
from evalmarket.langsmith.models import choose_model
|
|
12
|
+
from evalmarket.langsmith.playground import offer_langsmith_playground
|
|
13
|
+
from evalmarket.langsmith.prompts import import_langsmith_task
|
|
14
|
+
from evalmarket.langsmith.resources import langsmith_id
|
|
15
|
+
from evalmarket.langsmith.validation import langsmith_task
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def import_package(folder, dataset_name=None, profile=None, model_config=None):
|
|
19
|
+
folder = Path(folder).resolve()
|
|
20
|
+
model_path = Path(model_config).resolve() if model_config else None
|
|
21
|
+
config = json.loads((folder / "evaluator.json").read_text(encoding="utf-8-sig"))
|
|
22
|
+
if (
|
|
23
|
+
not isinstance(config, dict)
|
|
24
|
+
or not isinstance(config.get("name"), str)
|
|
25
|
+
or not config["name"].strip()
|
|
26
|
+
):
|
|
27
|
+
raise ValueError("evaluator.json needs a nonempty name.")
|
|
28
|
+
if dataset_name is None:
|
|
29
|
+
dataset_name = (
|
|
30
|
+
f"{config['name'].strip()} - {datetime.now(timezone.utc):%Y%m%d-%H%M%S-%f}"
|
|
31
|
+
)
|
|
32
|
+
judge = config.get("judge_type")
|
|
33
|
+
required = [folder / "README.md", folder / "dataset.json"]
|
|
34
|
+
if judge == "llm":
|
|
35
|
+
required += [
|
|
36
|
+
folder / name for name in ("prompt.json", "rubric.json", "variables.json")
|
|
37
|
+
]
|
|
38
|
+
if model_path:
|
|
39
|
+
required.append(model_path)
|
|
40
|
+
elif isinstance(judge, str) and judge in {"deterministic", "code"}:
|
|
41
|
+
filename, function = config.get("file"), config.get("function")
|
|
42
|
+
if (
|
|
43
|
+
filename not in ("evaluator.py", "evaluator.js", "evaluator.ts")
|
|
44
|
+
or not isinstance(function, str)
|
|
45
|
+
or not function.isidentifier()
|
|
46
|
+
):
|
|
47
|
+
raise ValueError(
|
|
48
|
+
"Declare a supported evaluator file and function in evaluator.json."
|
|
49
|
+
)
|
|
50
|
+
required.append(folder / filename)
|
|
51
|
+
evaluator = [
|
|
52
|
+
"evaluator",
|
|
53
|
+
"upload",
|
|
54
|
+
str(folder / filename),
|
|
55
|
+
"--function",
|
|
56
|
+
function,
|
|
57
|
+
]
|
|
58
|
+
else:
|
|
59
|
+
raise ValueError("judge_type must be deterministic, llm, or code.")
|
|
60
|
+
native = {}
|
|
61
|
+
for path in required:
|
|
62
|
+
if not path.is_file() or not path.stat().st_size:
|
|
63
|
+
raise ValueError(f"Required file is missing or empty: {path}")
|
|
64
|
+
if path.suffix == ".json" or path == model_path:
|
|
65
|
+
native[path] = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
66
|
+
|
|
67
|
+
task_path = folder / "task.json"
|
|
68
|
+
task = None
|
|
69
|
+
if task_path.exists() or task_path.is_symlink():
|
|
70
|
+
if (
|
|
71
|
+
task_path.is_symlink()
|
|
72
|
+
or not task_path.is_file()
|
|
73
|
+
or not 0 < task_path.stat().st_size <= MAX_NATIVE_FILE_BYTES
|
|
74
|
+
):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
"langchain/task.json must be a regular nonempty file up to 5 MiB."
|
|
77
|
+
)
|
|
78
|
+
try:
|
|
79
|
+
task = langsmith_task(
|
|
80
|
+
native_json(task_path.read_text(encoding="utf-8-sig")),
|
|
81
|
+
native[folder / "dataset.json"],
|
|
82
|
+
)
|
|
83
|
+
except (ValueError, RecursionError) as error:
|
|
84
|
+
raise ValueError(f"Invalid langchain/task.json: {error}") from None
|
|
85
|
+
|
|
86
|
+
if judge == "llm":
|
|
87
|
+
prompt = native[folder / "prompt.json"]
|
|
88
|
+
if not isinstance(prompt, list) or not prompt:
|
|
89
|
+
raise ValueError("prompt.json needs a nonempty array of messages.")
|
|
90
|
+
messages = [
|
|
91
|
+
[m.get("role"), m.get("content")] if isinstance(m, dict) else m
|
|
92
|
+
for m in prompt
|
|
93
|
+
]
|
|
94
|
+
if any(
|
|
95
|
+
not isinstance(m, list)
|
|
96
|
+
or len(m) != 2
|
|
97
|
+
or not all(isinstance(v, str) and v.strip() for v in m)
|
|
98
|
+
for m in messages
|
|
99
|
+
):
|
|
100
|
+
raise ValueError(
|
|
101
|
+
"prompt.json needs [role, content] pairs or message objects."
|
|
102
|
+
)
|
|
103
|
+
schema = native[folder / "rubric.json"]
|
|
104
|
+
variables = native[folder / "variables.json"]
|
|
105
|
+
if (
|
|
106
|
+
not isinstance(schema, dict)
|
|
107
|
+
or not schema
|
|
108
|
+
or not isinstance(variables, dict)
|
|
109
|
+
):
|
|
110
|
+
raise ValueError("rubric.json and variables.json need native JSON objects.")
|
|
111
|
+
if any(not isinstance(value, str) for value in variables.values()):
|
|
112
|
+
raise ValueError("variables.json must map names to field-path strings.")
|
|
113
|
+
# Match the official CLI's native schema envelope; preserve seller fields.
|
|
114
|
+
if "parameters" not in schema:
|
|
115
|
+
schema = {"name": "eval", "description": "", "parameters": schema}
|
|
116
|
+
elif not isinstance(schema["parameters"], dict):
|
|
117
|
+
raise ValueError(
|
|
118
|
+
"rubric.json parameters must contain a JSON schema object."
|
|
119
|
+
)
|
|
120
|
+
structured = {
|
|
121
|
+
"prompt": messages,
|
|
122
|
+
"schema": schema,
|
|
123
|
+
"variable_mapping": variables,
|
|
124
|
+
"template_format": "mustache",
|
|
125
|
+
}
|
|
126
|
+
if model_path:
|
|
127
|
+
model = native[model_path]
|
|
128
|
+
if not isinstance(model, dict) or not model:
|
|
129
|
+
raise ValueError(
|
|
130
|
+
"The model configuration must be a nonempty JSON object."
|
|
131
|
+
)
|
|
132
|
+
structured["model"] = model
|
|
133
|
+
|
|
134
|
+
executable = shutil.which("langsmith")
|
|
135
|
+
if not executable:
|
|
136
|
+
raise ValueError(
|
|
137
|
+
"Install the official LangSmith CLI, then run this importer again: "
|
|
138
|
+
"https://docs.langchain.com/langsmith/langsmith-cli"
|
|
139
|
+
)
|
|
140
|
+
command = [executable, "--format", "json"] + (
|
|
141
|
+
["--profile", profile] if profile else []
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def run(arguments, body=None):
|
|
145
|
+
# Keep files unchanged and delegate authentication/import behavior to the CLI.
|
|
146
|
+
try:
|
|
147
|
+
result = subprocess.run(
|
|
148
|
+
command + arguments,
|
|
149
|
+
check=True,
|
|
150
|
+
stdout=subprocess.PIPE,
|
|
151
|
+
text=True,
|
|
152
|
+
encoding="utf-8",
|
|
153
|
+
input=json.dumps(body) if body is not None else None,
|
|
154
|
+
)
|
|
155
|
+
except subprocess.CalledProcessError as error:
|
|
156
|
+
# Raw API errors are printed on stdout by the CLI. Preserve them
|
|
157
|
+
# so the user can see why an import failed, not just its exit code.
|
|
158
|
+
if error.stdout:
|
|
159
|
+
print(error.stdout, file=sys.stderr)
|
|
160
|
+
raise
|
|
161
|
+
return json.loads(result.stdout)
|
|
162
|
+
|
|
163
|
+
auth = run(["auth", "info"])
|
|
164
|
+
if not isinstance(auth, dict) or not isinstance(auth.get("authenticated"), bool):
|
|
165
|
+
raise ValueError("Could not read the LangSmith CLI's authentication state.")
|
|
166
|
+
if not auth["authenticated"] or (
|
|
167
|
+
auth.get("oauth_expired") is True and not auth.get("oauth_refresh_token")
|
|
168
|
+
):
|
|
169
|
+
print(
|
|
170
|
+
"Sign in to LangSmith in your browser to continue the import.", flush=True
|
|
171
|
+
)
|
|
172
|
+
login = [executable, "--format", "pretty"] + (
|
|
173
|
+
["--profile", profile] if profile else []
|
|
174
|
+
)
|
|
175
|
+
# Inherit the terminal so the official CLI can show its login link and
|
|
176
|
+
# handle workspace selection. It stores and manages its own login.
|
|
177
|
+
subprocess.run(login + ["auth", "login", "--prompt-workspace"], check=True)
|
|
178
|
+
auth = run(["auth", "info"])
|
|
179
|
+
if not isinstance(auth, dict) or auth.get("authenticated") is not True:
|
|
180
|
+
raise ValueError(
|
|
181
|
+
"LangSmith sign-in did not complete. Nothing has been uploaded."
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
if judge == "llm" and not model_path:
|
|
185
|
+
# Reuse the saved native configuration unchanged. The API requires both
|
|
186
|
+
# its model settings and ID; no model file or provider conversion is needed.
|
|
187
|
+
structured.update(choose_model(run))
|
|
188
|
+
|
|
189
|
+
print(
|
|
190
|
+
f"Uploading dataset {dataset_name!r} using your current LangSmith account...",
|
|
191
|
+
flush=True,
|
|
192
|
+
)
|
|
193
|
+
dataset = run(
|
|
194
|
+
["dataset", "upload", str(folder / "dataset.json"), "--name", dataset_name]
|
|
195
|
+
)
|
|
196
|
+
dataset_id = dataset.get("dataset_id") if isinstance(dataset, dict) else None
|
|
197
|
+
if not isinstance(dataset_id, str) or not dataset_id:
|
|
198
|
+
raise ValueError(
|
|
199
|
+
"The CLI did not return a dataset ID. Inspect LangSmith before retrying."
|
|
200
|
+
)
|
|
201
|
+
print(f"Created dataset: {dataset_id}", flush=True)
|
|
202
|
+
# Zero sampling installs the evaluator without automatically grading new runs.
|
|
203
|
+
if judge == "llm":
|
|
204
|
+
# The CLI's create-llm command requires a file; its authenticated API
|
|
205
|
+
# command also supports LangSmith's native saved-model reference.
|
|
206
|
+
evaluator_result = run(
|
|
207
|
+
["api", "runs/rules", "--method", "POST", "--input", "-"],
|
|
208
|
+
{
|
|
209
|
+
"display_name": config["name"],
|
|
210
|
+
"dataset_id": dataset_id,
|
|
211
|
+
"sampling_rate": 0,
|
|
212
|
+
"is_enabled": True,
|
|
213
|
+
"include_extended_stats": False,
|
|
214
|
+
"evaluators": [{"structured": structured}],
|
|
215
|
+
},
|
|
216
|
+
)
|
|
217
|
+
else:
|
|
218
|
+
evaluator_result = run(
|
|
219
|
+
evaluator
|
|
220
|
+
+ [
|
|
221
|
+
"--name",
|
|
222
|
+
config["name"],
|
|
223
|
+
"--dataset",
|
|
224
|
+
dataset_id,
|
|
225
|
+
"--sampling-rate",
|
|
226
|
+
"0",
|
|
227
|
+
]
|
|
228
|
+
)
|
|
229
|
+
print("Imported dataset and evaluator. No evaluation was run. Sampling is 0.")
|
|
230
|
+
if task is not None:
|
|
231
|
+
dataset_id = langsmith_id(dataset_id, "dataset")
|
|
232
|
+
field = "id" if judge == "llm" else "rule_id"
|
|
233
|
+
rule_id = langsmith_id(
|
|
234
|
+
evaluator_result.get(field) if isinstance(evaluator_result, dict) else None,
|
|
235
|
+
"evaluator rule",
|
|
236
|
+
)
|
|
237
|
+
print(f"Created evaluator rule: {rule_id}", flush=True)
|
|
238
|
+
url = import_langsmith_task(run, task, dataset_id, auth)
|
|
239
|
+
print("Import complete. No experiment has started.")
|
|
240
|
+
if url:
|
|
241
|
+
offer_langsmith_playground(run, url, dataset_id, rule_id)
|
|
242
|
+
else:
|
|
243
|
+
print(
|
|
244
|
+
"This package has no task.json for a prepared Playground. Ask the seller for the Playground task file, or follow the package's included task setup and enable grading in LangSmith when ready."
|
|
245
|
+
)
|
|
246
|
+
return dataset_id
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Langsmith models for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def choose_model(run):
|
|
5
|
+
records = run(["api", "playground-settings"])
|
|
6
|
+
if not isinstance(records, list):
|
|
7
|
+
raise ValueError("Could not read saved model configurations from LangSmith.")
|
|
8
|
+
models = [
|
|
9
|
+
record
|
|
10
|
+
for record in records
|
|
11
|
+
if isinstance(record, dict)
|
|
12
|
+
and isinstance(record.get("id"), str)
|
|
13
|
+
and record["id"]
|
|
14
|
+
and record.get("available_in_evaluators", True) is True
|
|
15
|
+
]
|
|
16
|
+
if not models:
|
|
17
|
+
raise ValueError(
|
|
18
|
+
"No saved models are available for evaluators in this LangSmith workspace. "
|
|
19
|
+
"Add or enable one in Settings > Model configurations, then rerun. "
|
|
20
|
+
"Nothing has been uploaded."
|
|
21
|
+
)
|
|
22
|
+
print("\nAvailable LangSmith judge models:")
|
|
23
|
+
for index, model in enumerate(models, 1):
|
|
24
|
+
# Only display the label and identity, never settings or credentials.
|
|
25
|
+
print(f" {index}. {model.get('name') or 'Unnamed model'!r} ({model['id']!r})")
|
|
26
|
+
while True:
|
|
27
|
+
try:
|
|
28
|
+
choice = input(
|
|
29
|
+
"Which model would you like to configure as your judge? "
|
|
30
|
+
f"[1-{len(models)}, q to cancel]: "
|
|
31
|
+
).strip()
|
|
32
|
+
except EOFError:
|
|
33
|
+
raise ValueError(
|
|
34
|
+
"Choose a model in an interactive terminal, or pass --model-config "
|
|
35
|
+
"with an existing native model file. Nothing has been uploaded."
|
|
36
|
+
) from None
|
|
37
|
+
if choice.lower() == "q":
|
|
38
|
+
raise ValueError("Import cancelled. Nothing has been uploaded.")
|
|
39
|
+
if choice.isascii() and choice.isdigit() and 1 <= int(choice) <= len(models):
|
|
40
|
+
selected = models[int(choice) - 1]
|
|
41
|
+
if (
|
|
42
|
+
not isinstance(selected.get("settings"), dict)
|
|
43
|
+
or not selected["settings"]
|
|
44
|
+
):
|
|
45
|
+
raise ValueError(
|
|
46
|
+
"The selected model has no saved configuration. Nothing has been uploaded."
|
|
47
|
+
)
|
|
48
|
+
return {
|
|
49
|
+
"playground_settings_id": selected["id"],
|
|
50
|
+
"model": selected["settings"],
|
|
51
|
+
}
|
|
52
|
+
print(f"Enter a number from 1 to {len(models)}, or q to cancel.")
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Langsmith playground for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import subprocess
|
|
4
|
+
import sys
|
|
5
|
+
import webbrowser
|
|
6
|
+
from urllib.parse import urlencode
|
|
7
|
+
|
|
8
|
+
from evalmarket.langsmith.resources import langsmith_id
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class PlaygroundSetupError(ValueError):
|
|
12
|
+
"""The import succeeded, but preparing the optional browser handoff failed."""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def offer_langsmith_playground(run, url, dataset_id, rule_id):
|
|
16
|
+
print(f"\nLangSmith Playground: {url}")
|
|
17
|
+
print(
|
|
18
|
+
"Review/select your answering model and click Start in LangSmith to run the experiment."
|
|
19
|
+
)
|
|
20
|
+
if not sys.stdin.isatty():
|
|
21
|
+
print(
|
|
22
|
+
"Unattended import: Playground not opened; enable the imported evaluator in LangSmith when ready."
|
|
23
|
+
)
|
|
24
|
+
return
|
|
25
|
+
print(
|
|
26
|
+
"Opening the Playground enables the imported evaluator for all cases in this dataset. Model calls after Start may incur charges."
|
|
27
|
+
)
|
|
28
|
+
try:
|
|
29
|
+
answer = (
|
|
30
|
+
input(
|
|
31
|
+
"Would you like to start an experiment in the LangSmith Playground? [y/N]: "
|
|
32
|
+
)
|
|
33
|
+
.strip()
|
|
34
|
+
.lower()
|
|
35
|
+
)
|
|
36
|
+
except (EOFError, KeyboardInterrupt):
|
|
37
|
+
print()
|
|
38
|
+
answer = ""
|
|
39
|
+
if answer not in {"y", "yes"}:
|
|
40
|
+
print(
|
|
41
|
+
"Playground skipped. Sampling remains 0; enable the imported evaluator when you are ready to run."
|
|
42
|
+
)
|
|
43
|
+
return
|
|
44
|
+
print(f"Preparing evaluator rule {rule_id} for dataset {dataset_id}...", flush=True)
|
|
45
|
+
try:
|
|
46
|
+
# Verify attachment before updating only the newly imported rule.
|
|
47
|
+
path = f"runs/rules/{rule_id}"
|
|
48
|
+
|
|
49
|
+
def read_rule():
|
|
50
|
+
# The native /{rule_id} endpoint has PATCH/DELETE, but no GET.
|
|
51
|
+
response = run(
|
|
52
|
+
[
|
|
53
|
+
"api",
|
|
54
|
+
"/api/v2/runs/rules?"
|
|
55
|
+
+ urlencode({"id": rule_id, "dataset_id": dataset_id}),
|
|
56
|
+
]
|
|
57
|
+
)
|
|
58
|
+
records = response.get("items") if isinstance(response, dict) else None
|
|
59
|
+
return (
|
|
60
|
+
records[0]
|
|
61
|
+
if isinstance(records, list)
|
|
62
|
+
and len(records) == 1
|
|
63
|
+
and response.get("next_cursor") is None
|
|
64
|
+
else None
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
rule = read_rule()
|
|
68
|
+
if (
|
|
69
|
+
not isinstance(rule, dict)
|
|
70
|
+
or rule.get("id") != rule_id
|
|
71
|
+
or rule.get("dataset_id") != dataset_id
|
|
72
|
+
or not isinstance(rule.get("display_name"), str)
|
|
73
|
+
or not rule["display_name"].strip()
|
|
74
|
+
):
|
|
75
|
+
raise ValueError(
|
|
76
|
+
"The imported evaluator's dataset attachment could not be confirmed."
|
|
77
|
+
)
|
|
78
|
+
evaluator_id = langsmith_id(rule.get("evaluator_id"), "evaluator")
|
|
79
|
+
run(
|
|
80
|
+
["api", path, "--method", "PATCH", "--input", "-"],
|
|
81
|
+
{
|
|
82
|
+
"display_name": rule["display_name"],
|
|
83
|
+
"dataset_id": dataset_id,
|
|
84
|
+
"evaluator_id": evaluator_id,
|
|
85
|
+
"sampling_rate": 1,
|
|
86
|
+
"is_enabled": True,
|
|
87
|
+
},
|
|
88
|
+
)
|
|
89
|
+
rule = read_rule()
|
|
90
|
+
if (
|
|
91
|
+
not isinstance(rule, dict)
|
|
92
|
+
or rule.get("id") != rule_id
|
|
93
|
+
or rule.get("dataset_id") != dataset_id
|
|
94
|
+
or rule.get("evaluator_id") != evaluator_id
|
|
95
|
+
or rule.get("sampling_rate") != 1
|
|
96
|
+
or rule.get("is_enabled") is not True
|
|
97
|
+
):
|
|
98
|
+
raise ValueError("Evaluator preparation could not be confirmed.")
|
|
99
|
+
except (
|
|
100
|
+
ValueError,
|
|
101
|
+
OSError,
|
|
102
|
+
subprocess.CalledProcessError,
|
|
103
|
+
EOFError,
|
|
104
|
+
KeyboardInterrupt,
|
|
105
|
+
) as error:
|
|
106
|
+
raise PlaygroundSetupError(
|
|
107
|
+
f"Dataset {dataset_id}, evaluator rule {rule_id}, and answering prompt remain imported. Evaluator preparation was not confirmed and may already have changed sampling; inspect it in LangSmith. No request was retried. No experiment was started. Use {url} after checking the evaluator; do not re-import."
|
|
108
|
+
) from error
|
|
109
|
+
try:
|
|
110
|
+
opened = webbrowser.open(url, new=2)
|
|
111
|
+
except (OSError, webbrowser.Error, KeyboardInterrupt):
|
|
112
|
+
opened = False
|
|
113
|
+
if opened:
|
|
114
|
+
print(
|
|
115
|
+
"Opened the Playground. No experiment has started; review the model and click Start in LangSmith."
|
|
116
|
+
)
|
|
117
|
+
else:
|
|
118
|
+
print(
|
|
119
|
+
f"Could not open a browser. Import is complete and the evaluator is enabled. Open this link manually: {url}"
|
|
120
|
+
)
|
|
121
|
+
print(
|
|
122
|
+
"No experiment has started. Review the model and click Start in LangSmith."
|
|
123
|
+
)
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Langsmith prompts for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from urllib.parse import urlencode
|
|
5
|
+
from uuid import uuid4
|
|
6
|
+
|
|
7
|
+
from evalmarket.langsmith.resources import langsmith_id
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def import_langsmith_task(run, manifest, dataset_id, auth):
|
|
11
|
+
"""Save a fresh private native prompt and verify it before offering the UI."""
|
|
12
|
+
handle = "evalmarket-task-" + uuid4().hex
|
|
13
|
+
print(f"Importing private answering prompt: {handle}", flush=True)
|
|
14
|
+
response = run(
|
|
15
|
+
["api", "repos", "--method", "POST", "--input", "-"],
|
|
16
|
+
{"repo_handle": handle, "is_public": False, "repo_type": "prompt"},
|
|
17
|
+
)
|
|
18
|
+
repo = response.get("repo") if isinstance(response, dict) else None
|
|
19
|
+
if (
|
|
20
|
+
not isinstance(repo, dict)
|
|
21
|
+
or repo.get("repo_handle") != handle
|
|
22
|
+
or repo.get("is_public") is not False
|
|
23
|
+
):
|
|
24
|
+
raise ValueError(
|
|
25
|
+
f"Could not confirm the new private prompt {handle}; inspect your workspace before retrying."
|
|
26
|
+
)
|
|
27
|
+
workspace = langsmith_id(repo.get("tenant_id"), "prompt workspace")
|
|
28
|
+
if auth.get("workspace_id") and workspace != langsmith_id(
|
|
29
|
+
auth["workspace_id"], "current workspace"
|
|
30
|
+
):
|
|
31
|
+
raise ValueError(
|
|
32
|
+
"The saved prompt belongs to a different workspace. Import stopped."
|
|
33
|
+
)
|
|
34
|
+
path = f"commits/-/{handle}"
|
|
35
|
+
response = run(
|
|
36
|
+
["api", path, "--method", "POST", "--input", "-"],
|
|
37
|
+
{"manifest": manifest, "skip_webhooks": True},
|
|
38
|
+
)
|
|
39
|
+
commit = response.get("commit") if isinstance(response, dict) else None
|
|
40
|
+
commit_hash = commit.get("commit_hash") if isinstance(commit, dict) else None
|
|
41
|
+
if not isinstance(commit_hash, str) or not re.fullmatch(
|
|
42
|
+
r"[a-fA-F0-9]{8,64}", commit_hash
|
|
43
|
+
):
|
|
44
|
+
raise ValueError(
|
|
45
|
+
f"No confirmed commit for prompt {handle}; inspect it before retrying."
|
|
46
|
+
)
|
|
47
|
+
saved = run(["api", f"{path}/{commit_hash}"])
|
|
48
|
+
if (
|
|
49
|
+
not isinstance(saved, dict)
|
|
50
|
+
or saved.get("commit_hash") != commit_hash
|
|
51
|
+
or saved.get("manifest") != manifest
|
|
52
|
+
):
|
|
53
|
+
raise ValueError(
|
|
54
|
+
f"Prompt {handle} readback did not match task.json. No Playground was opened."
|
|
55
|
+
)
|
|
56
|
+
print(f"Verified answering prompt: {handle} ({commit_hash})", flush=True)
|
|
57
|
+
origins = {
|
|
58
|
+
"https://api.smith.langchain.com": "https://smith.langchain.com",
|
|
59
|
+
"https://eu.api.smith.langchain.com": "https://eu.smith.langchain.com",
|
|
60
|
+
}
|
|
61
|
+
endpoint = str(auth.get("api_url", "")).rstrip("/")
|
|
62
|
+
if endpoint.endswith("/api/v1"):
|
|
63
|
+
endpoint = endpoint[:-7]
|
|
64
|
+
origin = origins.get(endpoint)
|
|
65
|
+
if not origin:
|
|
66
|
+
print(
|
|
67
|
+
"Your CLI endpoint has no known Playground browser origin. Open your LangSmith site, select the imported prompt and dataset, enable the evaluator, and review the model before clicking Start."
|
|
68
|
+
)
|
|
69
|
+
return None
|
|
70
|
+
return f"{origin}/o/{workspace}/prompts/{handle}/playground?" + urlencode(
|
|
71
|
+
{"commit": commit_hash, "datasetId": dataset_id}
|
|
72
|
+
)
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"""Langsmith resources for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
from uuid import UUID
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def langsmith_id(value, label):
|
|
7
|
+
try:
|
|
8
|
+
return str(UUID(value))
|
|
9
|
+
except (ValueError, TypeError, AttributeError):
|
|
10
|
+
raise ValueError(
|
|
11
|
+
f"LangSmith returned no valid {label} ID; inspect your workspace before retrying."
|
|
12
|
+
) from None
|