evalmarket-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,41 @@
1
+ """Isolate CLI startup before importing file-based modules or seller data."""
2
+
3
+ import sys
4
+
5
+
6
+ def restart_isolated(arguments):
7
+ """Restart with built-ins only, preserving arguments, environment and status."""
8
+ command = [sys.executable, "-I", *arguments]
9
+ if sys.platform == "win32":
10
+ # Windows execv loses the child's exit status. spawnv waits, but needs
11
+ # Windows CRT quoting. Do not import subprocess before isolation.
12
+ def quote_windows_argument(value):
13
+ quoted = '"'
14
+ backslashes = 0
15
+ for char in value:
16
+ if char == "\\":
17
+ backslashes += 1
18
+ continue
19
+ quoted += "\\" * (backslashes * 2 + 1 if char == '"' else backslashes)
20
+ quoted += char
21
+ backslashes = 0
22
+ return quoted + "\\" * (backslashes * 2) + '"'
23
+
24
+ native = __import__("nt")
25
+ raise SystemExit(
26
+ native.spawnv(
27
+ native.P_WAIT,
28
+ sys.executable,
29
+ [quote_windows_argument(arg) for arg in command],
30
+ )
31
+ )
32
+ __import__("posix").execv(sys.executable, command)
33
+
34
+
35
+ def main():
36
+ if not sys.flags.isolated:
37
+ restart_isolated(["-m", "evalmarket", *sys.argv[1:]])
38
+
39
+ from evalmarket.cli import main as run
40
+
41
+ return run()
@@ -0,0 +1,71 @@
1
+ """Json_data for native evaluation imports."""
2
+
3
+ import json
4
+ import math
5
+
6
+ MAX_NATIVE_FILE_BYTES = 5 * 1024 * 1024
7
+
8
+
9
+ def native_json(text):
10
+ """Reject duplicate keys and non-finite values, including exponent overflow."""
11
+
12
+ def pairs(items):
13
+ result = {}
14
+ for key, value in items:
15
+ if key in result:
16
+ raise ValueError("Duplicate JSON object key.")
17
+ result[key] = value
18
+ return result
19
+
20
+ value = json.loads(text, object_pairs_hook=pairs)
21
+
22
+ def finite(item):
23
+ if isinstance(item, float) and not math.isfinite(item):
24
+ raise ValueError("JSON numbers must be finite.")
25
+ if isinstance(item, dict):
26
+ for child in item.values():
27
+ finite(child)
28
+ elif isinstance(item, list):
29
+ for child in item:
30
+ finite(child)
31
+
32
+ finite(value)
33
+ return value
34
+
35
+
36
+ def same_json(left, right):
37
+ """JSON-aware equality distinguishes booleans from numbers at every depth."""
38
+ if isinstance(left, dict):
39
+ return (
40
+ isinstance(right, dict)
41
+ and left.keys() == right.keys()
42
+ and all(same_json(v, right[k]) for k, v in left.items())
43
+ )
44
+ if isinstance(left, list):
45
+ return (
46
+ isinstance(right, list)
47
+ and len(left) == len(right)
48
+ and all(same_json(a, b) for a, b in zip(left, right))
49
+ )
50
+ if type(left) in (int, float):
51
+ return type(right) in (int, float) and left == right
52
+ return type(left) is type(right) and left == right
53
+
54
+
55
+ def verify_fields(expected, actual, path="resource"):
56
+ """Supplied values must survive; provider-added optional null fields may appear."""
57
+ if isinstance(expected, dict) and isinstance(actual, dict):
58
+ if path not in {"resource", "dataset", "task", "scorer"}:
59
+ for key in actual.keys() - expected.keys():
60
+ if actual[key] is not None and not (
61
+ path.endswith(".function_data.data") and key == "code_hash"
62
+ ):
63
+ raise ValueError(
64
+ f"Readback mismatch at {path}; unexpected non-null fields."
65
+ )
66
+ for key, value in expected.items():
67
+ if key not in actual:
68
+ raise ValueError(f"Readback mismatch at {path}.{key} (missing field).")
69
+ verify_fields(value, actual[key], path + "." + key)
70
+ elif not same_json(expected, actual):
71
+ raise ValueError(f"Readback mismatch at {path}; import is not verified.")
@@ -0,0 +1 @@
1
+ """EvalMarket langsmith."""
@@ -0,0 +1,246 @@
1
+ """Langsmith importer for native evaluation imports."""
2
+
3
+ import json
4
+ import shutil
5
+ import subprocess
6
+ import sys
7
+ from datetime import datetime, timezone
8
+ from pathlib import Path
9
+
10
+ from evalmarket.json_data import MAX_NATIVE_FILE_BYTES, native_json
11
+ from evalmarket.langsmith.models import choose_model
12
+ from evalmarket.langsmith.playground import offer_langsmith_playground
13
+ from evalmarket.langsmith.prompts import import_langsmith_task
14
+ from evalmarket.langsmith.resources import langsmith_id
15
+ from evalmarket.langsmith.validation import langsmith_task
16
+
17
+
18
+ def import_package(folder, dataset_name=None, profile=None, model_config=None):
19
+ folder = Path(folder).resolve()
20
+ model_path = Path(model_config).resolve() if model_config else None
21
+ config = json.loads((folder / "evaluator.json").read_text(encoding="utf-8-sig"))
22
+ if (
23
+ not isinstance(config, dict)
24
+ or not isinstance(config.get("name"), str)
25
+ or not config["name"].strip()
26
+ ):
27
+ raise ValueError("evaluator.json needs a nonempty name.")
28
+ if dataset_name is None:
29
+ dataset_name = (
30
+ f"{config['name'].strip()} - {datetime.now(timezone.utc):%Y%m%d-%H%M%S-%f}"
31
+ )
32
+ judge = config.get("judge_type")
33
+ required = [folder / "README.md", folder / "dataset.json"]
34
+ if judge == "llm":
35
+ required += [
36
+ folder / name for name in ("prompt.json", "rubric.json", "variables.json")
37
+ ]
38
+ if model_path:
39
+ required.append(model_path)
40
+ elif isinstance(judge, str) and judge in {"deterministic", "code"}:
41
+ filename, function = config.get("file"), config.get("function")
42
+ if (
43
+ filename not in ("evaluator.py", "evaluator.js", "evaluator.ts")
44
+ or not isinstance(function, str)
45
+ or not function.isidentifier()
46
+ ):
47
+ raise ValueError(
48
+ "Declare a supported evaluator file and function in evaluator.json."
49
+ )
50
+ required.append(folder / filename)
51
+ evaluator = [
52
+ "evaluator",
53
+ "upload",
54
+ str(folder / filename),
55
+ "--function",
56
+ function,
57
+ ]
58
+ else:
59
+ raise ValueError("judge_type must be deterministic, llm, or code.")
60
+ native = {}
61
+ for path in required:
62
+ if not path.is_file() or not path.stat().st_size:
63
+ raise ValueError(f"Required file is missing or empty: {path}")
64
+ if path.suffix == ".json" or path == model_path:
65
+ native[path] = json.loads(path.read_text(encoding="utf-8-sig"))
66
+
67
+ task_path = folder / "task.json"
68
+ task = None
69
+ if task_path.exists() or task_path.is_symlink():
70
+ if (
71
+ task_path.is_symlink()
72
+ or not task_path.is_file()
73
+ or not 0 < task_path.stat().st_size <= MAX_NATIVE_FILE_BYTES
74
+ ):
75
+ raise ValueError(
76
+ "langchain/task.json must be a regular nonempty file up to 5 MiB."
77
+ )
78
+ try:
79
+ task = langsmith_task(
80
+ native_json(task_path.read_text(encoding="utf-8-sig")),
81
+ native[folder / "dataset.json"],
82
+ )
83
+ except (ValueError, RecursionError) as error:
84
+ raise ValueError(f"Invalid langchain/task.json: {error}") from None
85
+
86
+ if judge == "llm":
87
+ prompt = native[folder / "prompt.json"]
88
+ if not isinstance(prompt, list) or not prompt:
89
+ raise ValueError("prompt.json needs a nonempty array of messages.")
90
+ messages = [
91
+ [m.get("role"), m.get("content")] if isinstance(m, dict) else m
92
+ for m in prompt
93
+ ]
94
+ if any(
95
+ not isinstance(m, list)
96
+ or len(m) != 2
97
+ or not all(isinstance(v, str) and v.strip() for v in m)
98
+ for m in messages
99
+ ):
100
+ raise ValueError(
101
+ "prompt.json needs [role, content] pairs or message objects."
102
+ )
103
+ schema = native[folder / "rubric.json"]
104
+ variables = native[folder / "variables.json"]
105
+ if (
106
+ not isinstance(schema, dict)
107
+ or not schema
108
+ or not isinstance(variables, dict)
109
+ ):
110
+ raise ValueError("rubric.json and variables.json need native JSON objects.")
111
+ if any(not isinstance(value, str) for value in variables.values()):
112
+ raise ValueError("variables.json must map names to field-path strings.")
113
+ # Match the official CLI's native schema envelope; preserve seller fields.
114
+ if "parameters" not in schema:
115
+ schema = {"name": "eval", "description": "", "parameters": schema}
116
+ elif not isinstance(schema["parameters"], dict):
117
+ raise ValueError(
118
+ "rubric.json parameters must contain a JSON schema object."
119
+ )
120
+ structured = {
121
+ "prompt": messages,
122
+ "schema": schema,
123
+ "variable_mapping": variables,
124
+ "template_format": "mustache",
125
+ }
126
+ if model_path:
127
+ model = native[model_path]
128
+ if not isinstance(model, dict) or not model:
129
+ raise ValueError(
130
+ "The model configuration must be a nonempty JSON object."
131
+ )
132
+ structured["model"] = model
133
+
134
+ executable = shutil.which("langsmith")
135
+ if not executable:
136
+ raise ValueError(
137
+ "Install the official LangSmith CLI, then run this importer again: "
138
+ "https://docs.langchain.com/langsmith/langsmith-cli"
139
+ )
140
+ command = [executable, "--format", "json"] + (
141
+ ["--profile", profile] if profile else []
142
+ )
143
+
144
+ def run(arguments, body=None):
145
+ # Keep files unchanged and delegate authentication/import behavior to the CLI.
146
+ try:
147
+ result = subprocess.run(
148
+ command + arguments,
149
+ check=True,
150
+ stdout=subprocess.PIPE,
151
+ text=True,
152
+ encoding="utf-8",
153
+ input=json.dumps(body) if body is not None else None,
154
+ )
155
+ except subprocess.CalledProcessError as error:
156
+ # Raw API errors are printed on stdout by the CLI. Preserve them
157
+ # so the user can see why an import failed, not just its exit code.
158
+ if error.stdout:
159
+ print(error.stdout, file=sys.stderr)
160
+ raise
161
+ return json.loads(result.stdout)
162
+
163
+ auth = run(["auth", "info"])
164
+ if not isinstance(auth, dict) or not isinstance(auth.get("authenticated"), bool):
165
+ raise ValueError("Could not read the LangSmith CLI's authentication state.")
166
+ if not auth["authenticated"] or (
167
+ auth.get("oauth_expired") is True and not auth.get("oauth_refresh_token")
168
+ ):
169
+ print(
170
+ "Sign in to LangSmith in your browser to continue the import.", flush=True
171
+ )
172
+ login = [executable, "--format", "pretty"] + (
173
+ ["--profile", profile] if profile else []
174
+ )
175
+ # Inherit the terminal so the official CLI can show its login link and
176
+ # handle workspace selection. It stores and manages its own login.
177
+ subprocess.run(login + ["auth", "login", "--prompt-workspace"], check=True)
178
+ auth = run(["auth", "info"])
179
+ if not isinstance(auth, dict) or auth.get("authenticated") is not True:
180
+ raise ValueError(
181
+ "LangSmith sign-in did not complete. Nothing has been uploaded."
182
+ )
183
+
184
+ if judge == "llm" and not model_path:
185
+ # Reuse the saved native configuration unchanged. The API requires both
186
+ # its model settings and ID; no model file or provider conversion is needed.
187
+ structured.update(choose_model(run))
188
+
189
+ print(
190
+ f"Uploading dataset {dataset_name!r} using your current LangSmith account...",
191
+ flush=True,
192
+ )
193
+ dataset = run(
194
+ ["dataset", "upload", str(folder / "dataset.json"), "--name", dataset_name]
195
+ )
196
+ dataset_id = dataset.get("dataset_id") if isinstance(dataset, dict) else None
197
+ if not isinstance(dataset_id, str) or not dataset_id:
198
+ raise ValueError(
199
+ "The CLI did not return a dataset ID. Inspect LangSmith before retrying."
200
+ )
201
+ print(f"Created dataset: {dataset_id}", flush=True)
202
+ # Zero sampling installs the evaluator without automatically grading new runs.
203
+ if judge == "llm":
204
+ # The CLI's create-llm command requires a file; its authenticated API
205
+ # command also supports LangSmith's native saved-model reference.
206
+ evaluator_result = run(
207
+ ["api", "runs/rules", "--method", "POST", "--input", "-"],
208
+ {
209
+ "display_name": config["name"],
210
+ "dataset_id": dataset_id,
211
+ "sampling_rate": 0,
212
+ "is_enabled": True,
213
+ "include_extended_stats": False,
214
+ "evaluators": [{"structured": structured}],
215
+ },
216
+ )
217
+ else:
218
+ evaluator_result = run(
219
+ evaluator
220
+ + [
221
+ "--name",
222
+ config["name"],
223
+ "--dataset",
224
+ dataset_id,
225
+ "--sampling-rate",
226
+ "0",
227
+ ]
228
+ )
229
+ print("Imported dataset and evaluator. No evaluation was run. Sampling is 0.")
230
+ if task is not None:
231
+ dataset_id = langsmith_id(dataset_id, "dataset")
232
+ field = "id" if judge == "llm" else "rule_id"
233
+ rule_id = langsmith_id(
234
+ evaluator_result.get(field) if isinstance(evaluator_result, dict) else None,
235
+ "evaluator rule",
236
+ )
237
+ print(f"Created evaluator rule: {rule_id}", flush=True)
238
+ url = import_langsmith_task(run, task, dataset_id, auth)
239
+ print("Import complete. No experiment has started.")
240
+ if url:
241
+ offer_langsmith_playground(run, url, dataset_id, rule_id)
242
+ else:
243
+ print(
244
+ "This package has no task.json for a prepared Playground. Ask the seller for the Playground task file, or follow the package's included task setup and enable grading in LangSmith when ready."
245
+ )
246
+ return dataset_id
@@ -0,0 +1,52 @@
1
+ """Langsmith models for native evaluation imports."""
2
+
3
+
4
+ def choose_model(run):
5
+ records = run(["api", "playground-settings"])
6
+ if not isinstance(records, list):
7
+ raise ValueError("Could not read saved model configurations from LangSmith.")
8
+ models = [
9
+ record
10
+ for record in records
11
+ if isinstance(record, dict)
12
+ and isinstance(record.get("id"), str)
13
+ and record["id"]
14
+ and record.get("available_in_evaluators", True) is True
15
+ ]
16
+ if not models:
17
+ raise ValueError(
18
+ "No saved models are available for evaluators in this LangSmith workspace. "
19
+ "Add or enable one in Settings > Model configurations, then rerun. "
20
+ "Nothing has been uploaded."
21
+ )
22
+ print("\nAvailable LangSmith judge models:")
23
+ for index, model in enumerate(models, 1):
24
+ # Only display the label and identity, never settings or credentials.
25
+ print(f" {index}. {model.get('name') or 'Unnamed model'!r} ({model['id']!r})")
26
+ while True:
27
+ try:
28
+ choice = input(
29
+ "Which model would you like to configure as your judge? "
30
+ f"[1-{len(models)}, q to cancel]: "
31
+ ).strip()
32
+ except EOFError:
33
+ raise ValueError(
34
+ "Choose a model in an interactive terminal, or pass --model-config "
35
+ "with an existing native model file. Nothing has been uploaded."
36
+ ) from None
37
+ if choice.lower() == "q":
38
+ raise ValueError("Import cancelled. Nothing has been uploaded.")
39
+ if choice.isascii() and choice.isdigit() and 1 <= int(choice) <= len(models):
40
+ selected = models[int(choice) - 1]
41
+ if (
42
+ not isinstance(selected.get("settings"), dict)
43
+ or not selected["settings"]
44
+ ):
45
+ raise ValueError(
46
+ "The selected model has no saved configuration. Nothing has been uploaded."
47
+ )
48
+ return {
49
+ "playground_settings_id": selected["id"],
50
+ "model": selected["settings"],
51
+ }
52
+ print(f"Enter a number from 1 to {len(models)}, or q to cancel.")
@@ -0,0 +1,123 @@
1
+ """Langsmith playground for native evaluation imports."""
2
+
3
+ import subprocess
4
+ import sys
5
+ import webbrowser
6
+ from urllib.parse import urlencode
7
+
8
+ from evalmarket.langsmith.resources import langsmith_id
9
+
10
+
11
+ class PlaygroundSetupError(ValueError):
12
+ """The import succeeded, but preparing the optional browser handoff failed."""
13
+
14
+
15
+ def offer_langsmith_playground(run, url, dataset_id, rule_id):
16
+ print(f"\nLangSmith Playground: {url}")
17
+ print(
18
+ "Review/select your answering model and click Start in LangSmith to run the experiment."
19
+ )
20
+ if not sys.stdin.isatty():
21
+ print(
22
+ "Unattended import: Playground not opened; enable the imported evaluator in LangSmith when ready."
23
+ )
24
+ return
25
+ print(
26
+ "Opening the Playground enables the imported evaluator for all cases in this dataset. Model calls after Start may incur charges."
27
+ )
28
+ try:
29
+ answer = (
30
+ input(
31
+ "Would you like to start an experiment in the LangSmith Playground? [y/N]: "
32
+ )
33
+ .strip()
34
+ .lower()
35
+ )
36
+ except (EOFError, KeyboardInterrupt):
37
+ print()
38
+ answer = ""
39
+ if answer not in {"y", "yes"}:
40
+ print(
41
+ "Playground skipped. Sampling remains 0; enable the imported evaluator when you are ready to run."
42
+ )
43
+ return
44
+ print(f"Preparing evaluator rule {rule_id} for dataset {dataset_id}...", flush=True)
45
+ try:
46
+ # Verify attachment before updating only the newly imported rule.
47
+ path = f"runs/rules/{rule_id}"
48
+
49
+ def read_rule():
50
+ # The native /{rule_id} endpoint has PATCH/DELETE, but no GET.
51
+ response = run(
52
+ [
53
+ "api",
54
+ "/api/v2/runs/rules?"
55
+ + urlencode({"id": rule_id, "dataset_id": dataset_id}),
56
+ ]
57
+ )
58
+ records = response.get("items") if isinstance(response, dict) else None
59
+ return (
60
+ records[0]
61
+ if isinstance(records, list)
62
+ and len(records) == 1
63
+ and response.get("next_cursor") is None
64
+ else None
65
+ )
66
+
67
+ rule = read_rule()
68
+ if (
69
+ not isinstance(rule, dict)
70
+ or rule.get("id") != rule_id
71
+ or rule.get("dataset_id") != dataset_id
72
+ or not isinstance(rule.get("display_name"), str)
73
+ or not rule["display_name"].strip()
74
+ ):
75
+ raise ValueError(
76
+ "The imported evaluator's dataset attachment could not be confirmed."
77
+ )
78
+ evaluator_id = langsmith_id(rule.get("evaluator_id"), "evaluator")
79
+ run(
80
+ ["api", path, "--method", "PATCH", "--input", "-"],
81
+ {
82
+ "display_name": rule["display_name"],
83
+ "dataset_id": dataset_id,
84
+ "evaluator_id": evaluator_id,
85
+ "sampling_rate": 1,
86
+ "is_enabled": True,
87
+ },
88
+ )
89
+ rule = read_rule()
90
+ if (
91
+ not isinstance(rule, dict)
92
+ or rule.get("id") != rule_id
93
+ or rule.get("dataset_id") != dataset_id
94
+ or rule.get("evaluator_id") != evaluator_id
95
+ or rule.get("sampling_rate") != 1
96
+ or rule.get("is_enabled") is not True
97
+ ):
98
+ raise ValueError("Evaluator preparation could not be confirmed.")
99
+ except (
100
+ ValueError,
101
+ OSError,
102
+ subprocess.CalledProcessError,
103
+ EOFError,
104
+ KeyboardInterrupt,
105
+ ) as error:
106
+ raise PlaygroundSetupError(
107
+ f"Dataset {dataset_id}, evaluator rule {rule_id}, and answering prompt remain imported. Evaluator preparation was not confirmed and may already have changed sampling; inspect it in LangSmith. No request was retried. No experiment was started. Use {url} after checking the evaluator; do not re-import."
108
+ ) from error
109
+ try:
110
+ opened = webbrowser.open(url, new=2)
111
+ except (OSError, webbrowser.Error, KeyboardInterrupt):
112
+ opened = False
113
+ if opened:
114
+ print(
115
+ "Opened the Playground. No experiment has started; review the model and click Start in LangSmith."
116
+ )
117
+ else:
118
+ print(
119
+ f"Could not open a browser. Import is complete and the evaluator is enabled. Open this link manually: {url}"
120
+ )
121
+ print(
122
+ "No experiment has started. Review the model and click Start in LangSmith."
123
+ )
@@ -0,0 +1,72 @@
1
+ """Langsmith prompts for native evaluation imports."""
2
+
3
+ import re
4
+ from urllib.parse import urlencode
5
+ from uuid import uuid4
6
+
7
+ from evalmarket.langsmith.resources import langsmith_id
8
+
9
+
10
+ def import_langsmith_task(run, manifest, dataset_id, auth):
11
+ """Save a fresh private native prompt and verify it before offering the UI."""
12
+ handle = "evalmarket-task-" + uuid4().hex
13
+ print(f"Importing private answering prompt: {handle}", flush=True)
14
+ response = run(
15
+ ["api", "repos", "--method", "POST", "--input", "-"],
16
+ {"repo_handle": handle, "is_public": False, "repo_type": "prompt"},
17
+ )
18
+ repo = response.get("repo") if isinstance(response, dict) else None
19
+ if (
20
+ not isinstance(repo, dict)
21
+ or repo.get("repo_handle") != handle
22
+ or repo.get("is_public") is not False
23
+ ):
24
+ raise ValueError(
25
+ f"Could not confirm the new private prompt {handle}; inspect your workspace before retrying."
26
+ )
27
+ workspace = langsmith_id(repo.get("tenant_id"), "prompt workspace")
28
+ if auth.get("workspace_id") and workspace != langsmith_id(
29
+ auth["workspace_id"], "current workspace"
30
+ ):
31
+ raise ValueError(
32
+ "The saved prompt belongs to a different workspace. Import stopped."
33
+ )
34
+ path = f"commits/-/{handle}"
35
+ response = run(
36
+ ["api", path, "--method", "POST", "--input", "-"],
37
+ {"manifest": manifest, "skip_webhooks": True},
38
+ )
39
+ commit = response.get("commit") if isinstance(response, dict) else None
40
+ commit_hash = commit.get("commit_hash") if isinstance(commit, dict) else None
41
+ if not isinstance(commit_hash, str) or not re.fullmatch(
42
+ r"[a-fA-F0-9]{8,64}", commit_hash
43
+ ):
44
+ raise ValueError(
45
+ f"No confirmed commit for prompt {handle}; inspect it before retrying."
46
+ )
47
+ saved = run(["api", f"{path}/{commit_hash}"])
48
+ if (
49
+ not isinstance(saved, dict)
50
+ or saved.get("commit_hash") != commit_hash
51
+ or saved.get("manifest") != manifest
52
+ ):
53
+ raise ValueError(
54
+ f"Prompt {handle} readback did not match task.json. No Playground was opened."
55
+ )
56
+ print(f"Verified answering prompt: {handle} ({commit_hash})", flush=True)
57
+ origins = {
58
+ "https://api.smith.langchain.com": "https://smith.langchain.com",
59
+ "https://eu.api.smith.langchain.com": "https://eu.smith.langchain.com",
60
+ }
61
+ endpoint = str(auth.get("api_url", "")).rstrip("/")
62
+ if endpoint.endswith("/api/v1"):
63
+ endpoint = endpoint[:-7]
64
+ origin = origins.get(endpoint)
65
+ if not origin:
66
+ print(
67
+ "Your CLI endpoint has no known Playground browser origin. Open your LangSmith site, select the imported prompt and dataset, enable the evaluator, and review the model before clicking Start."
68
+ )
69
+ return None
70
+ return f"{origin}/o/{workspace}/prompts/{handle}/playground?" + urlencode(
71
+ {"commit": commit_hash, "datasetId": dataset_id}
72
+ )
@@ -0,0 +1,12 @@
1
+ """Langsmith resources for native evaluation imports."""
2
+
3
+ from uuid import UUID
4
+
5
+
6
+ def langsmith_id(value, label):
7
+ try:
8
+ return str(UUID(value))
9
+ except (ValueError, TypeError, AttributeError):
10
+ raise ValueError(
11
+ f"LangSmith returned no valid {label} ID; inspect your workspace before retrying."
12
+ ) from None