evalmarket-cli 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,253 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ !apps/app/public/downloads/
16
+ apps/app/public/downloads/*
17
+ !apps/app/public/downloads/evalmarket_cli-*-py3-none-any.whl
18
+ eggs/
19
+ .eggs/
20
+ lib/
21
+ lib64/
22
+ parts/
23
+ sdist/
24
+ var/
25
+ wheels/
26
+ share/python-wheels/
27
+ *.egg-info/
28
+ .installed.cfg
29
+ *.egg
30
+ MANIFEST
31
+
32
+ # PyInstaller
33
+ # Usually these files are written by a python script from a template
34
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
35
+ *.manifest
36
+ *.spec
37
+
38
+ # Installer logs
39
+ pip-log.txt
40
+ pip-delete-this-directory.txt
41
+
42
+ # Unit test / coverage reports
43
+ htmlcov/
44
+ .tox/
45
+ .nox/
46
+ .coverage
47
+ .coverage.*
48
+ .cache
49
+ nosetests.xml
50
+ coverage.xml
51
+ *.cover
52
+ *.py.cover
53
+ *.lcov
54
+ .hypothesis/
55
+ .pytest_cache/
56
+ pytest-cache-files-*/
57
+ cover/
58
+
59
+ # Translations
60
+ *.mo
61
+ *.pot
62
+
63
+ # Django stuff:
64
+ *.log
65
+ local_settings.py
66
+ db.sqlite3
67
+ db.sqlite3-journal
68
+
69
+ # Flask stuff:
70
+ instance/
71
+ .webassets-cache
72
+
73
+ # Scrapy stuff:
74
+ .scrapy
75
+
76
+ # Sphinx documentation
77
+ docs/_build/
78
+
79
+ # PyBuilder
80
+ .pybuilder/
81
+ target/
82
+
83
+ # Jupyter Notebook
84
+ .ipynb_checkpoints
85
+
86
+ # IPython
87
+ profile_default/
88
+ ipython_config.py
89
+
90
+ # pyenv
91
+ # For a library or package, you might want to ignore these files since the code is
92
+ # intended to run in multiple environments; otherwise, check them in:
93
+ # .python-version
94
+
95
+ # pipenv
96
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
97
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
98
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
99
+ # install all needed dependencies.
100
+ # Pipfile.lock
101
+
102
+ # uv
103
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
104
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
105
+ # commonly ignored for libraries.
106
+ # uv.lock
107
+
108
+ # poetry
109
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
110
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
111
+ # commonly ignored for libraries.
112
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
113
+ # poetry.lock
114
+ # poetry.toml
115
+
116
+ # pdm
117
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
118
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
119
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
120
+ # pdm.lock
121
+ # pdm.toml
122
+ .pdm-python
123
+ .pdm-build/
124
+
125
+ # pixi
126
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
127
+ # pixi.lock
128
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
129
+ # in the .venv directory. It is recommended not to include this directory in version control.
130
+ .pixi/*
131
+ !.pixi/config.toml
132
+
133
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
134
+ __pypackages__/
135
+
136
+ # Celery stuff
137
+ celerybeat-schedule*
138
+ celerybeat.pid
139
+
140
+ # Redis
141
+ *.rdb
142
+ *.aof
143
+ *.pid
144
+
145
+ # RabbitMQ
146
+ mnesia/
147
+ rabbitmq/
148
+ rabbitmq-data/
149
+
150
+ # ActiveMQ
151
+ activemq-data/
152
+
153
+ # SageMath parsed files
154
+ *.sage.py
155
+
156
+ # Environments
157
+ .env
158
+ .envrc
159
+ .venv
160
+ env/
161
+ venv/
162
+ ENV/
163
+ env.bak/
164
+ venv.bak/
165
+
166
+ # Spyder project settings
167
+ .spyderproject
168
+ .spyproject
169
+
170
+ # Rope project settings
171
+ .ropeproject
172
+
173
+ # mkdocs/Zensical documentation
174
+ /site
175
+
176
+ # mypy
177
+ .mypy_cache/
178
+ .dmypy.json
179
+ dmypy.json
180
+
181
+ # Pyre type checker
182
+ .pyre/
183
+
184
+ # pytype static type analyzer
185
+ .pytype/
186
+
187
+ # Cython debug symbols
188
+ cython_debug/
189
+
190
+ # PyCharm
191
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
192
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
193
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
194
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
195
+ # .idea/
196
+
197
+ # Abstra
198
+ # Abstra is an AI-powered process automation framework.
199
+ # Ignore directories containing user credentials, local state, and settings.
200
+ # Learn more at https://abstra.io/docs
201
+ .abstra/
202
+
203
+ # Visual Studio Code
204
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that
205
+ # can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
206
+ # and can be added to the global gitignore or merged into this file. However, if you prefer, you
207
+ # could uncomment the following to ignore the entire vscode folder
208
+ # .vscode/
209
+ # Temporary file for partial code execution
210
+ tempCodeRunnerFile.py
211
+
212
+ # Ruff stuff:
213
+ .ruff_cache/
214
+
215
+ # PyPI configuration file
216
+ .pypirc
217
+
218
+ # Marimo
219
+ marimo/_static/
220
+ marimo/_lsp/
221
+ __marimo__/
222
+
223
+ # Streamlit
224
+ .streamlit/secrets.toml
225
+
226
+ # Monorepo local and generated artifacts
227
+ .DS_Store
228
+ node_modules/
229
+ .next/
230
+ .nuxt/
231
+ .turbo/
232
+ coverage/
233
+ .env.*
234
+ !.env.example
235
+ !.env.*.example
236
+ *.tsbuildinfo
237
+
238
+ # Cloudflare / OpenNext local and generated output
239
+ .wrangler/
240
+ .open-next/
241
+
242
+ .dev.vars
243
+ .dev.vars.*
244
+
245
+ # The seller Build step is source code, not generated build output.
246
+ !apps/app/app/**/build/
247
+ !apps/app/app/**/build/**
248
+
249
+ # Isolated delivery worktrees and synthetic test scratch
250
+ .worktrees/
251
+ .test-tmp*/
252
+ .cache/
253
+ **/api/staging_eval_data/
@@ -0,0 +1,69 @@
1
+ Metadata-Version: 2.5
2
+ Name: evalmarket-cli
3
+ Version: 0.1.0
4
+ Summary: Import purchased EvalMarket evaluations into your evaluation platform
5
+ Requires-Python: >=3.10
6
+ Description-Content-Type: text/markdown
7
+
8
+ # EvalMarket Python importer
9
+
10
+ Customers install the versioned Python wheel from the marketplace's [CLI download page](https://www.evalmarket.ai/import-guide/cli). This page and the wheel ship together with the website. Python 3.10+ is required. The distribution has no Python runtime dependencies. LangSmith imports also need the official LangSmith CLI on PATH and its authentication/profile setup. Braintrust uses a local API key.
11
+
12
+ Download the wheel into a folder outside the extracted evaluation. In an activated Python virtual environment, run these commands from the wheel's folder:
13
+
14
+ ```sh
15
+ python -I -m pip install --no-index --no-deps --upgrade ./evalmarket_cli-0.1.0-py3-none-any.whl
16
+ evalmarket --version
17
+ evalmarket import /path/to/braintrust
18
+ evalmarket import /path/to/langchain --profile my-profile
19
+ ```
20
+
21
+ For upgrades, download the newer wheel and repeat the install with its exact filename in the same environment. Existing extracted evaluation folders can be reused. The package has not been published to a registry.
22
+
23
+ For development in the API environment, run `uv sync --locked` in `apps/api`,
24
+ then `uv run evalmarket import --help`.
25
+ The module entry point is `python -I -m evalmarket` in an installed environment.
26
+
27
+ The CLI preserves existing model selection, confirmation, readback verification,
28
+ partial-failure handling, and explicit experiment/Playground opt-in. It imports
29
+ an extracted platform folder. It does not download purchases or unpack ZIPs.
30
+
31
+ ## Source layout
32
+
33
+ ```text
34
+ src/evalmarket/
35
+ cli.py Argument parsing and error presentation
36
+ entrypoint.py Isolated process startup
37
+ json_data.py Shared file-size limit, strict JSON parsing and readback comparisons
38
+ braintrust/
39
+ validation.py Pure native package validation shared with the API
40
+ client.py HTTP, pagination, experiment-start acknowledgement
41
+ resources.py Resource identifiers and model labels
42
+ models.py Account model discovery and selection
43
+ importer.py Import orchestration and readback
44
+ experiments.py Explicit optional experiment start
45
+ langsmith/
46
+ validation.py Pure prompt-manifest validation shared with the API
47
+ resources.py Resource identifiers
48
+ models.py Saved judge-model selection
49
+ importer.py Native CLI import orchestration and authentication
50
+ prompts.py Private answering prompt creation and readback
51
+ playground.py Explicit optional browser handoff
52
+ ```
53
+
54
+ Use explicit module imports and keep pure validators independent of platform
55
+ clients. A new platform should have its own modules and explicit CLI routing;
56
+ there is no plugin framework or stable Python SDK contract in this release.
57
+ The version in `_version.py` drives distribution metadata and `--version`.
58
+
59
+ ## Build and verify
60
+
61
+ From the repository root, run:
62
+
63
+ ```sh
64
+ uv build --no-sources packages/evalmarket --out-dir dist
65
+ ```
66
+
67
+ Install the resulting wheel in a clean environment. Run `packages/evalmarket/tests/test_installed.py` with its Python interpreter. CI tests the installed CLI on Windows and Linux with Python 3.10 and 3.13. API tests exercise the provider workflows, ZIP delivery, and shared validators.
68
+
69
+ For a new customer release, bump the version, build the wheel and source distribution, and copy the reviewed wheel to `apps/app/public/downloads/` under its original filename. Update the download page and verify the public wheel before API cutover. Obtain release authorization before publication or deployment.
@@ -0,0 +1,62 @@
1
+ # EvalMarket Python importer
2
+
3
+ Customers install the versioned Python wheel from the marketplace's [CLI download page](https://www.evalmarket.ai/import-guide/cli). This page and the wheel ship together with the website. Python 3.10+ is required. The distribution has no Python runtime dependencies. LangSmith imports also need the official LangSmith CLI on PATH and its authentication/profile setup. Braintrust uses a local API key.
4
+
5
+ Download the wheel into a folder outside the extracted evaluation. In an activated Python virtual environment, run these commands from the wheel's folder:
6
+
7
+ ```sh
8
+ python -I -m pip install --no-index --no-deps --upgrade ./evalmarket_cli-0.1.0-py3-none-any.whl
9
+ evalmarket --version
10
+ evalmarket import /path/to/braintrust
11
+ evalmarket import /path/to/langchain --profile my-profile
12
+ ```
13
+
14
+ For upgrades, download the newer wheel and repeat the install with its exact filename in the same environment. Existing extracted evaluation folders can be reused. The package has not been published to a registry.
15
+
16
+ For development in the API environment, run `uv sync --locked` in `apps/api`,
17
+ then `uv run evalmarket import --help`.
18
+ The module entry point is `python -I -m evalmarket` in an installed environment.
19
+
20
+ The CLI preserves existing model selection, confirmation, readback verification,
21
+ partial-failure handling, and explicit experiment/Playground opt-in. It imports
22
+ an extracted platform folder. It does not download purchases or unpack ZIPs.
23
+
24
+ ## Source layout
25
+
26
+ ```text
27
+ src/evalmarket/
28
+ cli.py Argument parsing and error presentation
29
+ entrypoint.py Isolated process startup
30
+ json_data.py Shared file-size limit, strict JSON parsing and readback comparisons
31
+ braintrust/
32
+ validation.py Pure native package validation shared with the API
33
+ client.py HTTP, pagination, experiment-start acknowledgement
34
+ resources.py Resource identifiers and model labels
35
+ models.py Account model discovery and selection
36
+ importer.py Import orchestration and readback
37
+ experiments.py Explicit optional experiment start
38
+ langsmith/
39
+ validation.py Pure prompt-manifest validation shared with the API
40
+ resources.py Resource identifiers
41
+ models.py Saved judge-model selection
42
+ importer.py Native CLI import orchestration and authentication
43
+ prompts.py Private answering prompt creation and readback
44
+ playground.py Explicit optional browser handoff
45
+ ```
46
+
47
+ Use explicit module imports and keep pure validators independent of platform
48
+ clients. A new platform should have its own modules and explicit CLI routing;
49
+ there is no plugin framework or stable Python SDK contract in this release.
50
+ The version in `_version.py` drives distribution metadata and `--version`.
51
+
52
+ ## Build and verify
53
+
54
+ From the repository root, run:
55
+
56
+ ```sh
57
+ uv build --no-sources packages/evalmarket --out-dir dist
58
+ ```
59
+
60
+ Install the resulting wheel in a clean environment. Run `packages/evalmarket/tests/test_installed.py` with its Python interpreter. CI tests the installed CLI on Windows and Linux with Python 3.10 and 3.13. API tests exercise the provider workflows, ZIP delivery, and shared validators.
61
+
62
+ For a new customer release, bump the version, build the wheel and source distribution, and copy the reviewed wheel to `apps/app/public/downloads/` under its original filename. Update the download page and verify the public wheel before API cutover. Obtain release authorization before publication or deployment.
@@ -0,0 +1,33 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.27,<2"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "evalmarket-cli"
7
+ dynamic = ["version"]
8
+ description = "Import purchased EvalMarket evaluations into your evaluation platform"
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ dependencies = []
12
+
13
+ [project.scripts]
14
+ evalmarket = "evalmarket.entrypoint:main"
15
+
16
+ [tool.hatch.version]
17
+ path = "src/evalmarket/_version.py"
18
+
19
+ [tool.hatch.build.targets.wheel]
20
+ packages = ["src/evalmarket"]
21
+
22
+ [tool.hatch.build.targets.sdist]
23
+ include = ["/src", "/tests/test_installed.py", "/README.md", "/pyproject.toml"]
24
+
25
+ [tool.pytest.ini_options]
26
+ testpaths = ["tests"]
27
+
28
+ [tool.ruff]
29
+ target-version = "py310"
30
+
31
+ [tool.ruff.lint]
32
+ select = ["E", "F", "I", "B", "UP", "SIM"]
33
+ ignore = ["E501", "B905"]
@@ -0,0 +1 @@
1
+ """EvalMarket customer import tooling."""
@@ -0,0 +1,5 @@
1
+ """Support `python -I -m evalmarket` with the same CLI entry point."""
2
+
3
+ from evalmarket.entrypoint import main
4
+
5
+ raise SystemExit(main())
@@ -0,0 +1,3 @@
1
+ """Version used by the CLI and distribution metadata."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1 @@
1
+ """EvalMarket braintrust."""
@@ -0,0 +1,153 @@
1
+ """Braintrust client for native evaluation imports."""
2
+
3
+ import json
4
+ from time import monotonic
5
+ from urllib.error import HTTPError, URLError
6
+ from urllib.parse import urlencode, urlsplit
7
+ from urllib.request import HTTPRedirectHandler, Request, build_opener
8
+ from uuid import UUID
9
+
10
+ from evalmarket.braintrust.resources import model_label, resource_id
11
+ from evalmarket.json_data import native_json
12
+
13
+ BRAINTRUST_API = "https://api.braintrust.dev"
14
+
15
+
16
+ class NoCredentialRedirect(HTTPRedirectHandler):
17
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
18
+ raise ValueError("Braintrust redirect refused; credentials were not forwarded.")
19
+
20
+
21
+ class BraintrustClient:
22
+ def __init__(self, key):
23
+ self.key = key
24
+ self.opener = build_opener(NoCredentialRedirect())
25
+
26
+ def request(
27
+ self, method, path, body=None, *, _start_only=False, _model_scope=None, **query
28
+ ):
29
+ if not path.startswith("/v1/") or "?" in path or "#" in path:
30
+ raise ValueError("Unsupported Braintrust API path.")
31
+ headers = {
32
+ "Authorization": "Bearer " + self.key,
33
+ "Content-Type": "application/json",
34
+ "Accept": "text/event-stream" if _start_only else "application/json",
35
+ }
36
+ if _model_scope is not None:
37
+ project_id, provider_name = _model_scope
38
+ if (
39
+ method != "GET"
40
+ or path != "/v1/proxy/models"
41
+ or not model_label(provider_name)
42
+ ):
43
+ raise ValueError("Invalid Braintrust model discovery scope.")
44
+ headers.update(
45
+ {
46
+ "x-bt-project-id": str(UUID(project_id)),
47
+ "x-bt-endpoint-name": provider_name,
48
+ }
49
+ )
50
+ url = BRAINTRUST_API + path + ("?" + urlencode(query) if query else "")
51
+ request = Request(
52
+ url,
53
+ method=method,
54
+ headers=headers,
55
+ data=json.dumps(body, allow_nan=False).encode()
56
+ if body is not None
57
+ else None,
58
+ )
59
+ try:
60
+ with self.opener.open(request, timeout=60) as response:
61
+ if _start_only:
62
+ return read_experiment_start(response, body["project_id"])
63
+ return native_json(response.read().decode("utf-8"))
64
+ except HTTPError as error:
65
+ # Never echo provider bodies/URLs; these can include private data or credentials.
66
+ detail = {
67
+ 401: "invalid API key",
68
+ 403: "insufficient permissions",
69
+ 429: "rate limited",
70
+ }.get(error.code, "request rejected")
71
+ raise ValueError(
72
+ f"Braintrust HTTP {error.code}: {detail}. No write was retried."
73
+ ) from None
74
+ except (URLError, OSError, UnicodeError, ValueError) as error:
75
+ if isinstance(error, ValueError) and str(error).startswith(
76
+ "Braintrust redirect"
77
+ ):
78
+ raise
79
+ raise ValueError(
80
+ "Braintrust transport/read response failed; the outcome may be uncertain. No write was retried."
81
+ ) from None
82
+
83
+ def objects(self, kind, **filters):
84
+ cursor, seen = None, set()
85
+ while True:
86
+ page = self.request(
87
+ "GET",
88
+ "/v1/" + kind,
89
+ **filters,
90
+ limit=100,
91
+ **({"starting_after": cursor} if cursor else {}),
92
+ )
93
+ records = page.get("objects") if isinstance(page, dict) else None
94
+ if not isinstance(records, list):
95
+ raise ValueError("Braintrust returned an invalid object list.")
96
+ if not records:
97
+ return
98
+ for record in records:
99
+ identity = resource_id(record)
100
+ if identity in seen:
101
+ raise ValueError(
102
+ "Braintrust pagination repeated an object; stopped safely."
103
+ )
104
+ seen.add(identity)
105
+ yield record
106
+ cursor = resource_id(records[-1])
107
+
108
+
109
+ def read_experiment_start(response, project_id):
110
+ """Read only the SSE start acknowledgement, never results or completion."""
111
+ event, data, size = "", [], 0
112
+ deadline = monotonic() + 60
113
+ while monotonic() < deadline:
114
+ line = response.readline(65537)
115
+ size += len(line)
116
+ if not line or size > 65536:
117
+ break
118
+ line = line.decode("utf-8").rstrip("\r\n")
119
+ if not line:
120
+ if event == "start" and data:
121
+ started = native_json("\n".join(data))
122
+ if (
123
+ not isinstance(started, dict)
124
+ or started.get("projectId") != project_id
125
+ ):
126
+ break
127
+ identity = resource_id({"id": started.get("experimentId")})
128
+ url = started.get("experimentUrl")
129
+ if not isinstance(url, str) or any(
130
+ c.isspace() or ord(c) < 32 or ord(c) == 127 for c in url
131
+ ):
132
+ break
133
+ parsed = urlsplit(url)
134
+ if (
135
+ parsed.scheme != "https"
136
+ or parsed.netloc != "www.braintrust.dev"
137
+ or not parsed.path.startswith("/app/")
138
+ or "/experiments/" not in parsed.path
139
+ ):
140
+ break
141
+ return identity, url
142
+ if event or data:
143
+ # Do not parse error bodies or a result event as a successful start.
144
+ break
145
+ continue # SSE heartbeat/comment-only frame
146
+ field, separator, value = line.partition(":")
147
+ if separator and value.startswith(" "):
148
+ value = value[1:]
149
+ if field == "event":
150
+ event = value
151
+ elif field == "data":
152
+ data.append(value)
153
+ raise ValueError("No valid Braintrust experiment start acknowledgement.")
@@ -0,0 +1,67 @@
1
+ """Braintrust experiments for native evaluation imports."""
2
+
3
+ from uuid import uuid4
4
+
5
+
6
+ class ExperimentStartError(ValueError):
7
+ """An optional run failed after a successful, verified import."""
8
+
9
+
10
+ def offer_experiment(client, project_id, dataset_id, tasks, scorers, case_count):
11
+ print("\nStart an experiment")
12
+ print(
13
+ f" Runs one imported task on all {case_count} cases with the imported scorers."
14
+ )
15
+ print(
16
+ " Uses your Braintrust model/provider access; model calls may incur charges."
17
+ )
18
+ print(" Seller code runs with Braintrust hosted-function permissions.")
19
+ try:
20
+ answer = input("Would you like to start an experiment? [y/N]: ").strip().lower()
21
+ if answer not in {"y", "yes"}:
22
+ return False
23
+ selected = tasks[0]
24
+ if len(tasks) > 1:
25
+ choices = {str(i): task for i, task in enumerate(tasks, 1)}
26
+ while True:
27
+ print("\nChoose a task:")
28
+ for number, (task, _) in choices.items():
29
+ print(f" {number}. {task['name']!r}")
30
+ choice = input(f"Task number [1-{len(tasks)}, q to cancel]: ").strip()
31
+ if choice.lower() == "q":
32
+ return False
33
+ selected = choices.get(choice.lstrip("0"))
34
+ if selected is not None:
35
+ break
36
+ print(f"Enter a task number from 1 to {len(tasks)}, or q to cancel.")
37
+ except (EOFError, KeyboardInterrupt):
38
+ print()
39
+ return False
40
+ task, task_id = selected
41
+ name = "EvalMarket experiment " + uuid4().hex
42
+ print(f"\nStarting {name!r}", flush=True)
43
+ print(f" Task: {task['name']!r}", flush=True)
44
+ try:
45
+ identity, url = client.request(
46
+ "POST",
47
+ "/v1/eval",
48
+ {
49
+ "project_id": project_id,
50
+ "data": {"dataset_id": dataset_id},
51
+ "task": {"function_id": task_id},
52
+ "scores": [{"function_id": identity} for _, identity in scorers],
53
+ "experiment_name": name,
54
+ "stream": True,
55
+ "max_concurrency": 1,
56
+ "is_public": False,
57
+ },
58
+ _start_only=True,
59
+ )
60
+ except (ValueError, OSError, KeyboardInterrupt, EOFError):
61
+ raise ExperimentStartError(
62
+ f"Experiment start unconfirmed. Inspect project {project_id} for {name!r} before starting another run; it may already be running. No request was retried. Your imported resources are complete and do not need to be imported again."
63
+ ) from None
64
+ print(f"\nExperiment started. ID: {identity}")
65
+ print(f" {url}")
66
+ print("Braintrust will continue the run. Open the link for progress and results.")
67
+ return True