evalmarket-cli 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalmarket_cli-0.1.0/.gitignore +253 -0
- evalmarket_cli-0.1.0/PKG-INFO +69 -0
- evalmarket_cli-0.1.0/README.md +62 -0
- evalmarket_cli-0.1.0/pyproject.toml +33 -0
- evalmarket_cli-0.1.0/src/evalmarket/__init__.py +1 -0
- evalmarket_cli-0.1.0/src/evalmarket/__main__.py +5 -0
- evalmarket_cli-0.1.0/src/evalmarket/_version.py +3 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/__init__.py +1 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/client.py +153 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/experiments.py +67 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/importer.py +309 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/models.py +169 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/resources.py +21 -0
- evalmarket_cli-0.1.0/src/evalmarket/braintrust/validation.py +368 -0
- evalmarket_cli-0.1.0/src/evalmarket/cli.py +103 -0
- evalmarket_cli-0.1.0/src/evalmarket/entrypoint.py +41 -0
- evalmarket_cli-0.1.0/src/evalmarket/json_data.py +71 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/__init__.py +1 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/importer.py +246 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/models.py +52 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/playground.py +123 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/prompts.py +72 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/resources.py +12 -0
- evalmarket_cli-0.1.0/src/evalmarket/langsmith/validation.py +86 -0
- evalmarket_cli-0.1.0/tests/test_installed.py +220 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
!apps/app/public/downloads/
|
|
16
|
+
apps/app/public/downloads/*
|
|
17
|
+
!apps/app/public/downloads/evalmarket_cli-*-py3-none-any.whl
|
|
18
|
+
eggs/
|
|
19
|
+
.eggs/
|
|
20
|
+
lib/
|
|
21
|
+
lib64/
|
|
22
|
+
parts/
|
|
23
|
+
sdist/
|
|
24
|
+
var/
|
|
25
|
+
wheels/
|
|
26
|
+
share/python-wheels/
|
|
27
|
+
*.egg-info/
|
|
28
|
+
.installed.cfg
|
|
29
|
+
*.egg
|
|
30
|
+
MANIFEST
|
|
31
|
+
|
|
32
|
+
# PyInstaller
|
|
33
|
+
# Usually these files are written by a python script from a template
|
|
34
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
35
|
+
*.manifest
|
|
36
|
+
*.spec
|
|
37
|
+
|
|
38
|
+
# Installer logs
|
|
39
|
+
pip-log.txt
|
|
40
|
+
pip-delete-this-directory.txt
|
|
41
|
+
|
|
42
|
+
# Unit test / coverage reports
|
|
43
|
+
htmlcov/
|
|
44
|
+
.tox/
|
|
45
|
+
.nox/
|
|
46
|
+
.coverage
|
|
47
|
+
.coverage.*
|
|
48
|
+
.cache
|
|
49
|
+
nosetests.xml
|
|
50
|
+
coverage.xml
|
|
51
|
+
*.cover
|
|
52
|
+
*.py.cover
|
|
53
|
+
*.lcov
|
|
54
|
+
.hypothesis/
|
|
55
|
+
.pytest_cache/
|
|
56
|
+
pytest-cache-files-*/
|
|
57
|
+
cover/
|
|
58
|
+
|
|
59
|
+
# Translations
|
|
60
|
+
*.mo
|
|
61
|
+
*.pot
|
|
62
|
+
|
|
63
|
+
# Django stuff:
|
|
64
|
+
*.log
|
|
65
|
+
local_settings.py
|
|
66
|
+
db.sqlite3
|
|
67
|
+
db.sqlite3-journal
|
|
68
|
+
|
|
69
|
+
# Flask stuff:
|
|
70
|
+
instance/
|
|
71
|
+
.webassets-cache
|
|
72
|
+
|
|
73
|
+
# Scrapy stuff:
|
|
74
|
+
.scrapy
|
|
75
|
+
|
|
76
|
+
# Sphinx documentation
|
|
77
|
+
docs/_build/
|
|
78
|
+
|
|
79
|
+
# PyBuilder
|
|
80
|
+
.pybuilder/
|
|
81
|
+
target/
|
|
82
|
+
|
|
83
|
+
# Jupyter Notebook
|
|
84
|
+
.ipynb_checkpoints
|
|
85
|
+
|
|
86
|
+
# IPython
|
|
87
|
+
profile_default/
|
|
88
|
+
ipython_config.py
|
|
89
|
+
|
|
90
|
+
# pyenv
|
|
91
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
92
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
93
|
+
# .python-version
|
|
94
|
+
|
|
95
|
+
# pipenv
|
|
96
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
97
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
98
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
99
|
+
# install all needed dependencies.
|
|
100
|
+
# Pipfile.lock
|
|
101
|
+
|
|
102
|
+
# uv
|
|
103
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
104
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
105
|
+
# commonly ignored for libraries.
|
|
106
|
+
# uv.lock
|
|
107
|
+
|
|
108
|
+
# poetry
|
|
109
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
110
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
111
|
+
# commonly ignored for libraries.
|
|
112
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
113
|
+
# poetry.lock
|
|
114
|
+
# poetry.toml
|
|
115
|
+
|
|
116
|
+
# pdm
|
|
117
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
118
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
119
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
120
|
+
# pdm.lock
|
|
121
|
+
# pdm.toml
|
|
122
|
+
.pdm-python
|
|
123
|
+
.pdm-build/
|
|
124
|
+
|
|
125
|
+
# pixi
|
|
126
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
127
|
+
# pixi.lock
|
|
128
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
129
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
130
|
+
.pixi/*
|
|
131
|
+
!.pixi/config.toml
|
|
132
|
+
|
|
133
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
134
|
+
__pypackages__/
|
|
135
|
+
|
|
136
|
+
# Celery stuff
|
|
137
|
+
celerybeat-schedule*
|
|
138
|
+
celerybeat.pid
|
|
139
|
+
|
|
140
|
+
# Redis
|
|
141
|
+
*.rdb
|
|
142
|
+
*.aof
|
|
143
|
+
*.pid
|
|
144
|
+
|
|
145
|
+
# RabbitMQ
|
|
146
|
+
mnesia/
|
|
147
|
+
rabbitmq/
|
|
148
|
+
rabbitmq-data/
|
|
149
|
+
|
|
150
|
+
# ActiveMQ
|
|
151
|
+
activemq-data/
|
|
152
|
+
|
|
153
|
+
# SageMath parsed files
|
|
154
|
+
*.sage.py
|
|
155
|
+
|
|
156
|
+
# Environments
|
|
157
|
+
.env
|
|
158
|
+
.envrc
|
|
159
|
+
.venv
|
|
160
|
+
env/
|
|
161
|
+
venv/
|
|
162
|
+
ENV/
|
|
163
|
+
env.bak/
|
|
164
|
+
venv.bak/
|
|
165
|
+
|
|
166
|
+
# Spyder project settings
|
|
167
|
+
.spyderproject
|
|
168
|
+
.spyproject
|
|
169
|
+
|
|
170
|
+
# Rope project settings
|
|
171
|
+
.ropeproject
|
|
172
|
+
|
|
173
|
+
# mkdocs/Zensical documentation
|
|
174
|
+
/site
|
|
175
|
+
|
|
176
|
+
# mypy
|
|
177
|
+
.mypy_cache/
|
|
178
|
+
.dmypy.json
|
|
179
|
+
dmypy.json
|
|
180
|
+
|
|
181
|
+
# Pyre type checker
|
|
182
|
+
.pyre/
|
|
183
|
+
|
|
184
|
+
# pytype static type analyzer
|
|
185
|
+
.pytype/
|
|
186
|
+
|
|
187
|
+
# Cython debug symbols
|
|
188
|
+
cython_debug/
|
|
189
|
+
|
|
190
|
+
# PyCharm
|
|
191
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
192
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
193
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
194
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
195
|
+
# .idea/
|
|
196
|
+
|
|
197
|
+
# Abstra
|
|
198
|
+
# Abstra is an AI-powered process automation framework.
|
|
199
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
200
|
+
# Learn more at https://abstra.io/docs
|
|
201
|
+
.abstra/
|
|
202
|
+
|
|
203
|
+
# Visual Studio Code
|
|
204
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that
|
|
205
|
+
# can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
206
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer, you
|
|
207
|
+
# could uncomment the following to ignore the entire vscode folder
|
|
208
|
+
# .vscode/
|
|
209
|
+
# Temporary file for partial code execution
|
|
210
|
+
tempCodeRunnerFile.py
|
|
211
|
+
|
|
212
|
+
# Ruff stuff:
|
|
213
|
+
.ruff_cache/
|
|
214
|
+
|
|
215
|
+
# PyPI configuration file
|
|
216
|
+
.pypirc
|
|
217
|
+
|
|
218
|
+
# Marimo
|
|
219
|
+
marimo/_static/
|
|
220
|
+
marimo/_lsp/
|
|
221
|
+
__marimo__/
|
|
222
|
+
|
|
223
|
+
# Streamlit
|
|
224
|
+
.streamlit/secrets.toml
|
|
225
|
+
|
|
226
|
+
# Monorepo local and generated artifacts
|
|
227
|
+
.DS_Store
|
|
228
|
+
node_modules/
|
|
229
|
+
.next/
|
|
230
|
+
.nuxt/
|
|
231
|
+
.turbo/
|
|
232
|
+
coverage/
|
|
233
|
+
.env.*
|
|
234
|
+
!.env.example
|
|
235
|
+
!.env.*.example
|
|
236
|
+
*.tsbuildinfo
|
|
237
|
+
|
|
238
|
+
# Cloudflare / OpenNext local and generated output
|
|
239
|
+
.wrangler/
|
|
240
|
+
.open-next/
|
|
241
|
+
|
|
242
|
+
.dev.vars
|
|
243
|
+
.dev.vars.*
|
|
244
|
+
|
|
245
|
+
# The seller Build step is source code, not generated build output.
|
|
246
|
+
!apps/app/app/**/build/
|
|
247
|
+
!apps/app/app/**/build/**
|
|
248
|
+
|
|
249
|
+
# Isolated delivery worktrees and synthetic test scratch
|
|
250
|
+
.worktrees/
|
|
251
|
+
.test-tmp*/
|
|
252
|
+
.cache/
|
|
253
|
+
**/api/staging_eval_data/
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: evalmarket-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Import purchased EvalMarket evaluations into your evaluation platform
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
|
|
8
|
+
# EvalMarket Python importer
|
|
9
|
+
|
|
10
|
+
Customers install the versioned Python wheel from the marketplace's [CLI download page](https://www.evalmarket.ai/import-guide/cli). This page and the wheel ship together with the website. Python 3.10+ is required. The distribution has no Python runtime dependencies. LangSmith imports also need the official LangSmith CLI on PATH and its authentication/profile setup. Braintrust uses a local API key.
|
|
11
|
+
|
|
12
|
+
Download the wheel into a folder outside the extracted evaluation. In an activated Python virtual environment, run these commands from the wheel's folder:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
python -I -m pip install --no-index --no-deps --upgrade ./evalmarket_cli-0.1.0-py3-none-any.whl
|
|
16
|
+
evalmarket --version
|
|
17
|
+
evalmarket import /path/to/braintrust
|
|
18
|
+
evalmarket import /path/to/langchain --profile my-profile
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
For upgrades, download the newer wheel and repeat the install with its exact filename in the same environment. Existing extracted evaluation folders can be reused. The package has not been published to a registry.
|
|
22
|
+
|
|
23
|
+
For development in the API environment, run `uv sync --locked` in `apps/api`,
|
|
24
|
+
then `uv run evalmarket import --help`.
|
|
25
|
+
The module entry point is `python -I -m evalmarket` in an installed environment.
|
|
26
|
+
|
|
27
|
+
The CLI preserves existing model selection, confirmation, readback verification,
|
|
28
|
+
partial-failure handling, and explicit experiment/Playground opt-in. It imports
|
|
29
|
+
an extracted platform folder. It does not download purchases or unpack ZIPs.
|
|
30
|
+
|
|
31
|
+
## Source layout
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
src/evalmarket/
|
|
35
|
+
cli.py Argument parsing and error presentation
|
|
36
|
+
entrypoint.py Isolated process startup
|
|
37
|
+
json_data.py Shared file-size limit, strict JSON parsing and readback comparisons
|
|
38
|
+
braintrust/
|
|
39
|
+
validation.py Pure native package validation shared with the API
|
|
40
|
+
client.py HTTP, pagination, experiment-start acknowledgement
|
|
41
|
+
resources.py Resource identifiers and model labels
|
|
42
|
+
models.py Account model discovery and selection
|
|
43
|
+
importer.py Import orchestration and readback
|
|
44
|
+
experiments.py Explicit optional experiment start
|
|
45
|
+
langsmith/
|
|
46
|
+
validation.py Pure prompt-manifest validation shared with the API
|
|
47
|
+
resources.py Resource identifiers
|
|
48
|
+
models.py Saved judge-model selection
|
|
49
|
+
importer.py Native CLI import orchestration and authentication
|
|
50
|
+
prompts.py Private answering prompt creation and readback
|
|
51
|
+
playground.py Explicit optional browser handoff
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Use explicit module imports and keep pure validators independent of platform
|
|
55
|
+
clients. A new platform should have its own modules and explicit CLI routing;
|
|
56
|
+
there is no plugin framework or stable Python SDK contract in this release.
|
|
57
|
+
The version in `_version.py` drives distribution metadata and `--version`.
|
|
58
|
+
|
|
59
|
+
## Build and verify
|
|
60
|
+
|
|
61
|
+
From the repository root, run:
|
|
62
|
+
|
|
63
|
+
```sh
|
|
64
|
+
uv build --no-sources packages/evalmarket --out-dir dist
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Install the resulting wheel in a clean environment. Run `packages/evalmarket/tests/test_installed.py` with its Python interpreter. CI tests the installed CLI on Windows and Linux with Python 3.10 and 3.13. API tests exercise the provider workflows, ZIP delivery, and shared validators.
|
|
68
|
+
|
|
69
|
+
For a new customer release, bump the version, build the wheel and source distribution, and copy the reviewed wheel to `apps/app/public/downloads/` under its original filename. Update the download page and verify the public wheel before API cutover. Obtain release authorization before publication or deployment.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# EvalMarket Python importer
|
|
2
|
+
|
|
3
|
+
Customers install the versioned Python wheel from the marketplace's [CLI download page](https://www.evalmarket.ai/import-guide/cli). This page and the wheel ship together with the website. Python 3.10+ is required. The distribution has no Python runtime dependencies. LangSmith imports also need the official LangSmith CLI on PATH and its authentication/profile setup. Braintrust uses a local API key.
|
|
4
|
+
|
|
5
|
+
Download the wheel into a folder outside the extracted evaluation. In an activated Python virtual environment, run these commands from the wheel's folder:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
python -I -m pip install --no-index --no-deps --upgrade ./evalmarket_cli-0.1.0-py3-none-any.whl
|
|
9
|
+
evalmarket --version
|
|
10
|
+
evalmarket import /path/to/braintrust
|
|
11
|
+
evalmarket import /path/to/langchain --profile my-profile
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
For upgrades, download the newer wheel and repeat the install with its exact filename in the same environment. Existing extracted evaluation folders can be reused. The package has not been published to a registry.
|
|
15
|
+
|
|
16
|
+
For development in the API environment, run `uv sync --locked` in `apps/api`,
|
|
17
|
+
then `uv run evalmarket import --help`.
|
|
18
|
+
The module entry point is `python -I -m evalmarket` in an installed environment.
|
|
19
|
+
|
|
20
|
+
The CLI preserves existing model selection, confirmation, readback verification,
|
|
21
|
+
partial-failure handling, and explicit experiment/Playground opt-in. It imports
|
|
22
|
+
an extracted platform folder. It does not download purchases or unpack ZIPs.
|
|
23
|
+
|
|
24
|
+
## Source layout
|
|
25
|
+
|
|
26
|
+
```text
|
|
27
|
+
src/evalmarket/
|
|
28
|
+
cli.py Argument parsing and error presentation
|
|
29
|
+
entrypoint.py Isolated process startup
|
|
30
|
+
json_data.py Shared file-size limit, strict JSON parsing and readback comparisons
|
|
31
|
+
braintrust/
|
|
32
|
+
validation.py Pure native package validation shared with the API
|
|
33
|
+
client.py HTTP, pagination, experiment-start acknowledgement
|
|
34
|
+
resources.py Resource identifiers and model labels
|
|
35
|
+
models.py Account model discovery and selection
|
|
36
|
+
importer.py Import orchestration and readback
|
|
37
|
+
experiments.py Explicit optional experiment start
|
|
38
|
+
langsmith/
|
|
39
|
+
validation.py Pure prompt-manifest validation shared with the API
|
|
40
|
+
resources.py Resource identifiers
|
|
41
|
+
models.py Saved judge-model selection
|
|
42
|
+
importer.py Native CLI import orchestration and authentication
|
|
43
|
+
prompts.py Private answering prompt creation and readback
|
|
44
|
+
playground.py Explicit optional browser handoff
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Use explicit module imports and keep pure validators independent of platform
|
|
48
|
+
clients. A new platform should have its own modules and explicit CLI routing;
|
|
49
|
+
there is no plugin framework or stable Python SDK contract in this release.
|
|
50
|
+
The version in `_version.py` drives distribution metadata and `--version`.
|
|
51
|
+
|
|
52
|
+
## Build and verify
|
|
53
|
+
|
|
54
|
+
From the repository root, run:
|
|
55
|
+
|
|
56
|
+
```sh
|
|
57
|
+
uv build --no-sources packages/evalmarket --out-dir dist
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Install the resulting wheel in a clean environment. Run `packages/evalmarket/tests/test_installed.py` with its Python interpreter. CI tests the installed CLI on Windows and Linux with Python 3.10 and 3.13. API tests exercise the provider workflows, ZIP delivery, and shared validators.
|
|
61
|
+
|
|
62
|
+
For a new customer release, bump the version, build the wheel and source distribution, and copy the reviewed wheel to `apps/app/public/downloads/` under its original filename. Update the download page and verify the public wheel before API cutover. Obtain release authorization before publication or deployment.
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27,<2"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "evalmarket-cli"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Import purchased EvalMarket evaluations into your evaluation platform"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = []
|
|
12
|
+
|
|
13
|
+
[project.scripts]
|
|
14
|
+
evalmarket = "evalmarket.entrypoint:main"
|
|
15
|
+
|
|
16
|
+
[tool.hatch.version]
|
|
17
|
+
path = "src/evalmarket/_version.py"
|
|
18
|
+
|
|
19
|
+
[tool.hatch.build.targets.wheel]
|
|
20
|
+
packages = ["src/evalmarket"]
|
|
21
|
+
|
|
22
|
+
[tool.hatch.build.targets.sdist]
|
|
23
|
+
include = ["/src", "/tests/test_installed.py", "/README.md", "/pyproject.toml"]
|
|
24
|
+
|
|
25
|
+
[tool.pytest.ini_options]
|
|
26
|
+
testpaths = ["tests"]
|
|
27
|
+
|
|
28
|
+
[tool.ruff]
|
|
29
|
+
target-version = "py310"
|
|
30
|
+
|
|
31
|
+
[tool.ruff.lint]
|
|
32
|
+
select = ["E", "F", "I", "B", "UP", "SIM"]
|
|
33
|
+
ignore = ["E501", "B905"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""EvalMarket customer import tooling."""
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""EvalMarket braintrust."""
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Braintrust client for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from time import monotonic
|
|
5
|
+
from urllib.error import HTTPError, URLError
|
|
6
|
+
from urllib.parse import urlencode, urlsplit
|
|
7
|
+
from urllib.request import HTTPRedirectHandler, Request, build_opener
|
|
8
|
+
from uuid import UUID
|
|
9
|
+
|
|
10
|
+
from evalmarket.braintrust.resources import model_label, resource_id
|
|
11
|
+
from evalmarket.json_data import native_json
|
|
12
|
+
|
|
13
|
+
BRAINTRUST_API = "https://api.braintrust.dev"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class NoCredentialRedirect(HTTPRedirectHandler):
|
|
17
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
18
|
+
raise ValueError("Braintrust redirect refused; credentials were not forwarded.")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class BraintrustClient:
|
|
22
|
+
def __init__(self, key):
|
|
23
|
+
self.key = key
|
|
24
|
+
self.opener = build_opener(NoCredentialRedirect())
|
|
25
|
+
|
|
26
|
+
def request(
|
|
27
|
+
self, method, path, body=None, *, _start_only=False, _model_scope=None, **query
|
|
28
|
+
):
|
|
29
|
+
if not path.startswith("/v1/") or "?" in path or "#" in path:
|
|
30
|
+
raise ValueError("Unsupported Braintrust API path.")
|
|
31
|
+
headers = {
|
|
32
|
+
"Authorization": "Bearer " + self.key,
|
|
33
|
+
"Content-Type": "application/json",
|
|
34
|
+
"Accept": "text/event-stream" if _start_only else "application/json",
|
|
35
|
+
}
|
|
36
|
+
if _model_scope is not None:
|
|
37
|
+
project_id, provider_name = _model_scope
|
|
38
|
+
if (
|
|
39
|
+
method != "GET"
|
|
40
|
+
or path != "/v1/proxy/models"
|
|
41
|
+
or not model_label(provider_name)
|
|
42
|
+
):
|
|
43
|
+
raise ValueError("Invalid Braintrust model discovery scope.")
|
|
44
|
+
headers.update(
|
|
45
|
+
{
|
|
46
|
+
"x-bt-project-id": str(UUID(project_id)),
|
|
47
|
+
"x-bt-endpoint-name": provider_name,
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
url = BRAINTRUST_API + path + ("?" + urlencode(query) if query else "")
|
|
51
|
+
request = Request(
|
|
52
|
+
url,
|
|
53
|
+
method=method,
|
|
54
|
+
headers=headers,
|
|
55
|
+
data=json.dumps(body, allow_nan=False).encode()
|
|
56
|
+
if body is not None
|
|
57
|
+
else None,
|
|
58
|
+
)
|
|
59
|
+
try:
|
|
60
|
+
with self.opener.open(request, timeout=60) as response:
|
|
61
|
+
if _start_only:
|
|
62
|
+
return read_experiment_start(response, body["project_id"])
|
|
63
|
+
return native_json(response.read().decode("utf-8"))
|
|
64
|
+
except HTTPError as error:
|
|
65
|
+
# Never echo provider bodies/URLs; these can include private data or credentials.
|
|
66
|
+
detail = {
|
|
67
|
+
401: "invalid API key",
|
|
68
|
+
403: "insufficient permissions",
|
|
69
|
+
429: "rate limited",
|
|
70
|
+
}.get(error.code, "request rejected")
|
|
71
|
+
raise ValueError(
|
|
72
|
+
f"Braintrust HTTP {error.code}: {detail}. No write was retried."
|
|
73
|
+
) from None
|
|
74
|
+
except (URLError, OSError, UnicodeError, ValueError) as error:
|
|
75
|
+
if isinstance(error, ValueError) and str(error).startswith(
|
|
76
|
+
"Braintrust redirect"
|
|
77
|
+
):
|
|
78
|
+
raise
|
|
79
|
+
raise ValueError(
|
|
80
|
+
"Braintrust transport/read response failed; the outcome may be uncertain. No write was retried."
|
|
81
|
+
) from None
|
|
82
|
+
|
|
83
|
+
def objects(self, kind, **filters):
|
|
84
|
+
cursor, seen = None, set()
|
|
85
|
+
while True:
|
|
86
|
+
page = self.request(
|
|
87
|
+
"GET",
|
|
88
|
+
"/v1/" + kind,
|
|
89
|
+
**filters,
|
|
90
|
+
limit=100,
|
|
91
|
+
**({"starting_after": cursor} if cursor else {}),
|
|
92
|
+
)
|
|
93
|
+
records = page.get("objects") if isinstance(page, dict) else None
|
|
94
|
+
if not isinstance(records, list):
|
|
95
|
+
raise ValueError("Braintrust returned an invalid object list.")
|
|
96
|
+
if not records:
|
|
97
|
+
return
|
|
98
|
+
for record in records:
|
|
99
|
+
identity = resource_id(record)
|
|
100
|
+
if identity in seen:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"Braintrust pagination repeated an object; stopped safely."
|
|
103
|
+
)
|
|
104
|
+
seen.add(identity)
|
|
105
|
+
yield record
|
|
106
|
+
cursor = resource_id(records[-1])
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def read_experiment_start(response, project_id):
|
|
110
|
+
"""Read only the SSE start acknowledgement, never results or completion."""
|
|
111
|
+
event, data, size = "", [], 0
|
|
112
|
+
deadline = monotonic() + 60
|
|
113
|
+
while monotonic() < deadline:
|
|
114
|
+
line = response.readline(65537)
|
|
115
|
+
size += len(line)
|
|
116
|
+
if not line or size > 65536:
|
|
117
|
+
break
|
|
118
|
+
line = line.decode("utf-8").rstrip("\r\n")
|
|
119
|
+
if not line:
|
|
120
|
+
if event == "start" and data:
|
|
121
|
+
started = native_json("\n".join(data))
|
|
122
|
+
if (
|
|
123
|
+
not isinstance(started, dict)
|
|
124
|
+
or started.get("projectId") != project_id
|
|
125
|
+
):
|
|
126
|
+
break
|
|
127
|
+
identity = resource_id({"id": started.get("experimentId")})
|
|
128
|
+
url = started.get("experimentUrl")
|
|
129
|
+
if not isinstance(url, str) or any(
|
|
130
|
+
c.isspace() or ord(c) < 32 or ord(c) == 127 for c in url
|
|
131
|
+
):
|
|
132
|
+
break
|
|
133
|
+
parsed = urlsplit(url)
|
|
134
|
+
if (
|
|
135
|
+
parsed.scheme != "https"
|
|
136
|
+
or parsed.netloc != "www.braintrust.dev"
|
|
137
|
+
or not parsed.path.startswith("/app/")
|
|
138
|
+
or "/experiments/" not in parsed.path
|
|
139
|
+
):
|
|
140
|
+
break
|
|
141
|
+
return identity, url
|
|
142
|
+
if event or data:
|
|
143
|
+
# Do not parse error bodies or a result event as a successful start.
|
|
144
|
+
break
|
|
145
|
+
continue # SSE heartbeat/comment-only frame
|
|
146
|
+
field, separator, value = line.partition(":")
|
|
147
|
+
if separator and value.startswith(" "):
|
|
148
|
+
value = value[1:]
|
|
149
|
+
if field == "event":
|
|
150
|
+
event = value
|
|
151
|
+
elif field == "data":
|
|
152
|
+
data.append(value)
|
|
153
|
+
raise ValueError("No valid Braintrust experiment start acknowledgement.")
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Braintrust experiments for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
from uuid import uuid4
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class ExperimentStartError(ValueError):
|
|
7
|
+
"""An optional run failed after a successful, verified import."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def offer_experiment(client, project_id, dataset_id, tasks, scorers, case_count):
|
|
11
|
+
print("\nStart an experiment")
|
|
12
|
+
print(
|
|
13
|
+
f" Runs one imported task on all {case_count} cases with the imported scorers."
|
|
14
|
+
)
|
|
15
|
+
print(
|
|
16
|
+
" Uses your Braintrust model/provider access; model calls may incur charges."
|
|
17
|
+
)
|
|
18
|
+
print(" Seller code runs with Braintrust hosted-function permissions.")
|
|
19
|
+
try:
|
|
20
|
+
answer = input("Would you like to start an experiment? [y/N]: ").strip().lower()
|
|
21
|
+
if answer not in {"y", "yes"}:
|
|
22
|
+
return False
|
|
23
|
+
selected = tasks[0]
|
|
24
|
+
if len(tasks) > 1:
|
|
25
|
+
choices = {str(i): task for i, task in enumerate(tasks, 1)}
|
|
26
|
+
while True:
|
|
27
|
+
print("\nChoose a task:")
|
|
28
|
+
for number, (task, _) in choices.items():
|
|
29
|
+
print(f" {number}. {task['name']!r}")
|
|
30
|
+
choice = input(f"Task number [1-{len(tasks)}, q to cancel]: ").strip()
|
|
31
|
+
if choice.lower() == "q":
|
|
32
|
+
return False
|
|
33
|
+
selected = choices.get(choice.lstrip("0"))
|
|
34
|
+
if selected is not None:
|
|
35
|
+
break
|
|
36
|
+
print(f"Enter a task number from 1 to {len(tasks)}, or q to cancel.")
|
|
37
|
+
except (EOFError, KeyboardInterrupt):
|
|
38
|
+
print()
|
|
39
|
+
return False
|
|
40
|
+
task, task_id = selected
|
|
41
|
+
name = "EvalMarket experiment " + uuid4().hex
|
|
42
|
+
print(f"\nStarting {name!r}", flush=True)
|
|
43
|
+
print(f" Task: {task['name']!r}", flush=True)
|
|
44
|
+
try:
|
|
45
|
+
identity, url = client.request(
|
|
46
|
+
"POST",
|
|
47
|
+
"/v1/eval",
|
|
48
|
+
{
|
|
49
|
+
"project_id": project_id,
|
|
50
|
+
"data": {"dataset_id": dataset_id},
|
|
51
|
+
"task": {"function_id": task_id},
|
|
52
|
+
"scores": [{"function_id": identity} for _, identity in scorers],
|
|
53
|
+
"experiment_name": name,
|
|
54
|
+
"stream": True,
|
|
55
|
+
"max_concurrency": 1,
|
|
56
|
+
"is_public": False,
|
|
57
|
+
},
|
|
58
|
+
_start_only=True,
|
|
59
|
+
)
|
|
60
|
+
except (ValueError, OSError, KeyboardInterrupt, EOFError):
|
|
61
|
+
raise ExperimentStartError(
|
|
62
|
+
f"Experiment start unconfirmed. Inspect project {project_id} for {name!r} before starting another run; it may already be running. No request was retried. Your imported resources are complete and do not need to be imported again."
|
|
63
|
+
) from None
|
|
64
|
+
print(f"\nExperiment started. ID: {identity}")
|
|
65
|
+
print(f" {url}")
|
|
66
|
+
print("Braintrust will continue the run. Open the link for progress and results.")
|
|
67
|
+
return True
|