evalmarket-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- evalmarket/__init__.py +1 -0
- evalmarket/__main__.py +5 -0
- evalmarket/_version.py +3 -0
- evalmarket/braintrust/__init__.py +1 -0
- evalmarket/braintrust/client.py +153 -0
- evalmarket/braintrust/experiments.py +67 -0
- evalmarket/braintrust/importer.py +309 -0
- evalmarket/braintrust/models.py +169 -0
- evalmarket/braintrust/resources.py +21 -0
- evalmarket/braintrust/validation.py +368 -0
- evalmarket/cli.py +103 -0
- evalmarket/entrypoint.py +41 -0
- evalmarket/json_data.py +71 -0
- evalmarket/langsmith/__init__.py +1 -0
- evalmarket/langsmith/importer.py +246 -0
- evalmarket/langsmith/models.py +52 -0
- evalmarket/langsmith/playground.py +123 -0
- evalmarket/langsmith/prompts.py +72 -0
- evalmarket/langsmith/resources.py +12 -0
- evalmarket/langsmith/validation.py +86 -0
- evalmarket_cli-0.1.0.dist-info/METADATA +69 -0
- evalmarket_cli-0.1.0.dist-info/RECORD +24 -0
- evalmarket_cli-0.1.0.dist-info/WHEEL +4 -0
- evalmarket_cli-0.1.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Langsmith validation for native evaluation imports."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def langsmith_task(manifest, cases):
|
|
5
|
+
"""Check a native prompt without deserializing or executing seller objects."""
|
|
6
|
+
|
|
7
|
+
def fail(message):
|
|
8
|
+
raise ValueError(f"langchain/task.json: {message}")
|
|
9
|
+
|
|
10
|
+
prompt_paths = {
|
|
11
|
+
(namespace, "prompts", module, name)
|
|
12
|
+
for namespace in ("langchain", "langchain_core")
|
|
13
|
+
for module, name in (
|
|
14
|
+
("prompt", "PromptTemplate"),
|
|
15
|
+
("chat", "ChatPromptTemplate"),
|
|
16
|
+
)
|
|
17
|
+
} | {("langchain_core", "prompts", "structured", "StructuredPrompt")}
|
|
18
|
+
identity = manifest.get("id") if isinstance(manifest, dict) else None
|
|
19
|
+
if (
|
|
20
|
+
not isinstance(identity, list)
|
|
21
|
+
or len(identity) != 4
|
|
22
|
+
or not all(isinstance(part, str) for part in identity)
|
|
23
|
+
or tuple(identity) not in prompt_paths
|
|
24
|
+
or manifest.get("lc") != 1
|
|
25
|
+
or manifest.get("type") != "constructor"
|
|
26
|
+
or not isinstance(manifest.get("kwargs"), dict)
|
|
27
|
+
):
|
|
28
|
+
fail(
|
|
29
|
+
"supply a native serialized PromptTemplate, ChatPromptTemplate, or StructuredPrompt without a model or runnable code."
|
|
30
|
+
)
|
|
31
|
+
options = manifest["kwargs"]
|
|
32
|
+
variables = options.get("input_variables")
|
|
33
|
+
if (
|
|
34
|
+
not isinstance(variables, list)
|
|
35
|
+
or len(variables) > 15
|
|
36
|
+
or any(not isinstance(v, str) or not v.strip() for v in variables)
|
|
37
|
+
or len(set(variables)) != len(variables)
|
|
38
|
+
):
|
|
39
|
+
fail("input_variables must be an array of up to 15 unique nonempty names.")
|
|
40
|
+
if not isinstance(cases, list) or not cases:
|
|
41
|
+
fail("dataset.json must contain examples for the answering task.")
|
|
42
|
+
for index, case in enumerate(cases, 1):
|
|
43
|
+
inputs = case.get("inputs") if isinstance(case, dict) else None
|
|
44
|
+
if not isinstance(inputs, dict) or any(v not in inputs for v in variables):
|
|
45
|
+
fail(
|
|
46
|
+
f"declared task variables must come from every example's inputs (example {index}); reference outputs are not task inputs."
|
|
47
|
+
)
|
|
48
|
+
if identity[-1] == "PromptTemplate":
|
|
49
|
+
if (
|
|
50
|
+
not isinstance(options.get("template"), str)
|
|
51
|
+
or not options["template"].strip()
|
|
52
|
+
):
|
|
53
|
+
fail("PromptTemplate needs a nonempty template.")
|
|
54
|
+
elif not isinstance(options.get("messages"), list) or not options["messages"]:
|
|
55
|
+
fail("chat/structured prompts need a nonempty messages array.")
|
|
56
|
+
if identity[-1] == "StructuredPrompt" and (
|
|
57
|
+
not isinstance(options.get("schema_"), dict) or not options["schema_"]
|
|
58
|
+
):
|
|
59
|
+
fail("StructuredPrompt needs its nonempty native schema_ object.")
|
|
60
|
+
|
|
61
|
+
def check_objects(value):
|
|
62
|
+
if isinstance(value, dict):
|
|
63
|
+
if value.get("lc") == 1:
|
|
64
|
+
path = value.get("id")
|
|
65
|
+
if (
|
|
66
|
+
value.get("type") != "constructor"
|
|
67
|
+
or not isinstance(path, list)
|
|
68
|
+
or not all(isinstance(p, str) for p in path)
|
|
69
|
+
or len(path) < 3
|
|
70
|
+
or path[0] not in ("langchain", "langchain_core")
|
|
71
|
+
or not (
|
|
72
|
+
path[1] in ("prompts", "messages")
|
|
73
|
+
or path[1:3] == ["schema", "messages"]
|
|
74
|
+
)
|
|
75
|
+
):
|
|
76
|
+
fail(
|
|
77
|
+
"only native prompt/message objects are supported; omit models, secrets, and executable runnables."
|
|
78
|
+
)
|
|
79
|
+
for child in value.values():
|
|
80
|
+
check_objects(child)
|
|
81
|
+
elif isinstance(value, list):
|
|
82
|
+
for child in value:
|
|
83
|
+
check_objects(child)
|
|
84
|
+
|
|
85
|
+
check_objects(manifest)
|
|
86
|
+
return manifest
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: evalmarket-cli
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Import purchased EvalMarket evaluations into your evaluation platform
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
|
|
8
|
+
# EvalMarket Python importer
|
|
9
|
+
|
|
10
|
+
Customers install the versioned Python wheel from the marketplace's [CLI download page](https://www.evalmarket.ai/import-guide/cli). This page and the wheel ship together with the website. Python 3.10+ is required. The distribution has no Python runtime dependencies. LangSmith imports also need the official LangSmith CLI on PATH and its authentication/profile setup. Braintrust uses a local API key.
|
|
11
|
+
|
|
12
|
+
Download the wheel into a folder outside the extracted evaluation. In an activated Python virtual environment, run these commands from the wheel's folder:
|
|
13
|
+
|
|
14
|
+
```sh
|
|
15
|
+
python -I -m pip install --no-index --no-deps --upgrade ./evalmarket_cli-0.1.0-py3-none-any.whl
|
|
16
|
+
evalmarket --version
|
|
17
|
+
evalmarket import /path/to/braintrust
|
|
18
|
+
evalmarket import /path/to/langchain --profile my-profile
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
For upgrades, download the newer wheel and repeat the install with its exact filename in the same environment. Existing extracted evaluation folders can be reused. The package has not been published to a registry.
|
|
22
|
+
|
|
23
|
+
For development in the API environment, run `uv sync --locked` in `apps/api`,
|
|
24
|
+
then `uv run evalmarket import --help`.
|
|
25
|
+
The module entry point is `python -I -m evalmarket` in an installed environment.
|
|
26
|
+
|
|
27
|
+
The CLI preserves existing model selection, confirmation, readback verification,
|
|
28
|
+
partial-failure handling, and explicit experiment/Playground opt-in. It imports
|
|
29
|
+
an extracted platform folder. It does not download purchases or unpack ZIPs.
|
|
30
|
+
|
|
31
|
+
## Source layout
|
|
32
|
+
|
|
33
|
+
```text
|
|
34
|
+
src/evalmarket/
|
|
35
|
+
cli.py Argument parsing and error presentation
|
|
36
|
+
entrypoint.py Isolated process startup
|
|
37
|
+
json_data.py Shared file-size limit, strict JSON parsing and readback comparisons
|
|
38
|
+
braintrust/
|
|
39
|
+
validation.py Pure native package validation shared with the API
|
|
40
|
+
client.py HTTP, pagination, experiment-start acknowledgement
|
|
41
|
+
resources.py Resource identifiers and model labels
|
|
42
|
+
models.py Account model discovery and selection
|
|
43
|
+
importer.py Import orchestration and readback
|
|
44
|
+
experiments.py Explicit optional experiment start
|
|
45
|
+
langsmith/
|
|
46
|
+
validation.py Pure prompt-manifest validation shared with the API
|
|
47
|
+
resources.py Resource identifiers
|
|
48
|
+
models.py Saved judge-model selection
|
|
49
|
+
importer.py Native CLI import orchestration and authentication
|
|
50
|
+
prompts.py Private answering prompt creation and readback
|
|
51
|
+
playground.py Explicit optional browser handoff
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Use explicit module imports and keep pure validators independent of platform
|
|
55
|
+
clients. A new platform should have its own modules and explicit CLI routing;
|
|
56
|
+
there is no plugin framework or stable Python SDK contract in this release.
|
|
57
|
+
The version in `_version.py` drives distribution metadata and `--version`.
|
|
58
|
+
|
|
59
|
+
## Build and verify
|
|
60
|
+
|
|
61
|
+
From the repository root, run:
|
|
62
|
+
|
|
63
|
+
```sh
|
|
64
|
+
uv build --no-sources packages/evalmarket --out-dir dist
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Install the resulting wheel in a clean environment. Run `packages/evalmarket/tests/test_installed.py` with its Python interpreter. CI tests the installed CLI on Windows and Linux with Python 3.10 and 3.13. API tests exercise the provider workflows, ZIP delivery, and shared validators.
|
|
68
|
+
|
|
69
|
+
For a new customer release, bump the version, build the wheel and source distribution, and copy the reviewed wheel to `apps/app/public/downloads/` under its original filename. Update the download page and verify the public wheel before API cutover. Obtain release authorization before publication or deployment.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
evalmarket/__init__.py,sha256=RWKjZH_-BQM1Zn4fhrugqcdc32c10lGv2lrZzHoG9Q8,43
|
|
2
|
+
evalmarket/__main__.py,sha256=tuAkQDL92cONVwH6fr6OqweoO5XIG1P9ZvpkycF1mwY,142
|
|
3
|
+
evalmarket/_version.py,sha256=2fO2a2Q6wssbzc7aMxwWqqfaJgrTIGbt_qufCKSBhVE,80
|
|
4
|
+
evalmarket/cli.py,sha256=e5zYimX__34eWZm2ibYrW7UnydFbGTo-P2MEC1Lqid0,3756
|
|
5
|
+
evalmarket/entrypoint.py,sha256=zf9Ip1PtOqntoFXoAgixhuUzReceUPAY3Df7boLRfN8,1385
|
|
6
|
+
evalmarket/json_data.py,sha256=ZLOuxbeyuAESXHFUsAro_RQ9rty4vCFlGuT4AFxpBE4,2608
|
|
7
|
+
evalmarket/braintrust/__init__.py,sha256=Lv7I-ZODQwSoUk9nYUdqTeFSxYz5jpcYPPgYYfgrkRA,30
|
|
8
|
+
evalmarket/braintrust/client.py,sha256=2XQLKItzkrf7NQxz_q6rxKkDw45CDNJ5BdydRl9x7n0,6165
|
|
9
|
+
evalmarket/braintrust/experiments.py,sha256=QdksmqXrksLfQIyfK0fxrdYZoaEP_wIoibpAtO3cIqk,2842
|
|
10
|
+
evalmarket/braintrust/importer.py,sha256=3TUYW6Ro3XylJV2ihBZfu7MwDErOCsglTsxOIxRwX94,14400
|
|
11
|
+
evalmarket/braintrust/models.py,sha256=vp4b9C3fec6l42kD28vrhCuSLfGcxmcK22VlFAOTU1w,7879
|
|
12
|
+
evalmarket/braintrust/resources.py,sha256=HvTSytxOShtpTTMOIU6NgdUmblVDP-6uH7ggvQHDPNo,555
|
|
13
|
+
evalmarket/braintrust/validation.py,sha256=0L2u2ZO4_LriLzjJb5vy9AgVYP0Zv2w3IlTEKD7SW6s,15401
|
|
14
|
+
evalmarket/langsmith/__init__.py,sha256=8sSt0QJnv994qTXpkeqSYrccdnw87DXw6FRx034TQtE,29
|
|
15
|
+
evalmarket/langsmith/importer.py,sha256=gssH1X67p-vs-2Rp1xQbggt9lZ7FkdL-8LRXsL6v4-c,10351
|
|
16
|
+
evalmarket/langsmith/models.py,sha256=341GSs2OOng1JrBZwH1UXddNcjQ8NmKT9VnP60d0nNk,2264
|
|
17
|
+
evalmarket/langsmith/playground.py,sha256=_pyi6s1Z3f_F2HSHyzKQTp8vr5xSOIfSXUzpl4cc8dQ,4640
|
|
18
|
+
evalmarket/langsmith/prompts.py,sha256=bmcSIh555IQCGjSW3bmwLuzz9YLD4ZThBWpipSs8y8w,3055
|
|
19
|
+
evalmarket/langsmith/resources.py,sha256=EbS-V2NpFwIsiL08MZpoTQqHH9prpcqAFOtQCKbs5MM,360
|
|
20
|
+
evalmarket/langsmith/validation.py,sha256=q43sObNeb0jgFjBPuBEc30lYAOq3SW7_bFFBrsq2dl8,3742
|
|
21
|
+
evalmarket_cli-0.1.0.dist-info/METADATA,sha256=SExDeSXGUpETl0asNdPXQvqbUgDcRh-nMlVzHFWPWBI,3714
|
|
22
|
+
evalmarket_cli-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
23
|
+
evalmarket_cli-0.1.0.dist-info/entry_points.txt,sha256=0UC7i5Y-ozsCw4wCSjmyc9wCQh21SGPRCL7NoaHy7DY,58
|
|
24
|
+
evalmarket_cli-0.1.0.dist-info/RECORD,,
|