evalmarket-cli 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,86 @@
1
+ """Langsmith validation for native evaluation imports."""
2
+
3
+
4
+ def langsmith_task(manifest, cases):
5
+ """Check a native prompt without deserializing or executing seller objects."""
6
+
7
+ def fail(message):
8
+ raise ValueError(f"langchain/task.json: {message}")
9
+
10
+ prompt_paths = {
11
+ (namespace, "prompts", module, name)
12
+ for namespace in ("langchain", "langchain_core")
13
+ for module, name in (
14
+ ("prompt", "PromptTemplate"),
15
+ ("chat", "ChatPromptTemplate"),
16
+ )
17
+ } | {("langchain_core", "prompts", "structured", "StructuredPrompt")}
18
+ identity = manifest.get("id") if isinstance(manifest, dict) else None
19
+ if (
20
+ not isinstance(identity, list)
21
+ or len(identity) != 4
22
+ or not all(isinstance(part, str) for part in identity)
23
+ or tuple(identity) not in prompt_paths
24
+ or manifest.get("lc") != 1
25
+ or manifest.get("type") != "constructor"
26
+ or not isinstance(manifest.get("kwargs"), dict)
27
+ ):
28
+ fail(
29
+ "supply a native serialized PromptTemplate, ChatPromptTemplate, or StructuredPrompt without a model or runnable code."
30
+ )
31
+ options = manifest["kwargs"]
32
+ variables = options.get("input_variables")
33
+ if (
34
+ not isinstance(variables, list)
35
+ or len(variables) > 15
36
+ or any(not isinstance(v, str) or not v.strip() for v in variables)
37
+ or len(set(variables)) != len(variables)
38
+ ):
39
+ fail("input_variables must be an array of up to 15 unique nonempty names.")
40
+ if not isinstance(cases, list) or not cases:
41
+ fail("dataset.json must contain examples for the answering task.")
42
+ for index, case in enumerate(cases, 1):
43
+ inputs = case.get("inputs") if isinstance(case, dict) else None
44
+ if not isinstance(inputs, dict) or any(v not in inputs for v in variables):
45
+ fail(
46
+ f"declared task variables must come from every example's inputs (example {index}); reference outputs are not task inputs."
47
+ )
48
+ if identity[-1] == "PromptTemplate":
49
+ if (
50
+ not isinstance(options.get("template"), str)
51
+ or not options["template"].strip()
52
+ ):
53
+ fail("PromptTemplate needs a nonempty template.")
54
+ elif not isinstance(options.get("messages"), list) or not options["messages"]:
55
+ fail("chat/structured prompts need a nonempty messages array.")
56
+ if identity[-1] == "StructuredPrompt" and (
57
+ not isinstance(options.get("schema_"), dict) or not options["schema_"]
58
+ ):
59
+ fail("StructuredPrompt needs its nonempty native schema_ object.")
60
+
61
+ def check_objects(value):
62
+ if isinstance(value, dict):
63
+ if value.get("lc") == 1:
64
+ path = value.get("id")
65
+ if (
66
+ value.get("type") != "constructor"
67
+ or not isinstance(path, list)
68
+ or not all(isinstance(p, str) for p in path)
69
+ or len(path) < 3
70
+ or path[0] not in ("langchain", "langchain_core")
71
+ or not (
72
+ path[1] in ("prompts", "messages")
73
+ or path[1:3] == ["schema", "messages"]
74
+ )
75
+ ):
76
+ fail(
77
+ "only native prompt/message objects are supported; omit models, secrets, and executable runnables."
78
+ )
79
+ for child in value.values():
80
+ check_objects(child)
81
+ elif isinstance(value, list):
82
+ for child in value:
83
+ check_objects(child)
84
+
85
+ check_objects(manifest)
86
+ return manifest
@@ -0,0 +1,69 @@
1
+ Metadata-Version: 2.5
2
+ Name: evalmarket-cli
3
+ Version: 0.1.0
4
+ Summary: Import purchased EvalMarket evaluations into your evaluation platform
5
+ Requires-Python: >=3.10
6
+ Description-Content-Type: text/markdown
7
+
8
+ # EvalMarket Python importer
9
+
10
+ Customers install the versioned Python wheel from the marketplace's [CLI download page](https://www.evalmarket.ai/import-guide/cli). This page and the wheel ship together with the website. Python 3.10+ is required. The distribution has no Python runtime dependencies. LangSmith imports also need the official LangSmith CLI on PATH and its authentication/profile setup. Braintrust uses a local API key.
11
+
12
+ Download the wheel into a folder outside the extracted evaluation. In an activated Python virtual environment, run these commands from the wheel's folder:
13
+
14
+ ```sh
15
+ python -I -m pip install --no-index --no-deps --upgrade ./evalmarket_cli-0.1.0-py3-none-any.whl
16
+ evalmarket --version
17
+ evalmarket import /path/to/braintrust
18
+ evalmarket import /path/to/langchain --profile my-profile
19
+ ```
20
+
21
+ For upgrades, download the newer wheel and repeat the install with its exact filename in the same environment. Existing extracted evaluation folders can be reused. The package has not been published to a registry.
22
+
23
+ For development in the API environment, run `uv sync --locked` in `apps/api`,
24
+ then `uv run evalmarket import --help`.
25
+ The module entry point is `python -I -m evalmarket` in an installed environment.
26
+
27
+ The CLI preserves existing model selection, confirmation, readback verification,
28
+ partial-failure handling, and explicit experiment/Playground opt-in. It imports
29
+ an extracted platform folder. It does not download purchases or unpack ZIPs.
30
+
31
+ ## Source layout
32
+
33
+ ```text
34
+ src/evalmarket/
35
+ cli.py Argument parsing and error presentation
36
+ entrypoint.py Isolated process startup
37
+ json_data.py Shared file-size limit, strict JSON parsing and readback comparisons
38
+ braintrust/
39
+ validation.py Pure native package validation shared with the API
40
+ client.py HTTP, pagination, experiment-start acknowledgement
41
+ resources.py Resource identifiers and model labels
42
+ models.py Account model discovery and selection
43
+ importer.py Import orchestration and readback
44
+ experiments.py Explicit optional experiment start
45
+ langsmith/
46
+ validation.py Pure prompt-manifest validation shared with the API
47
+ resources.py Resource identifiers
48
+ models.py Saved judge-model selection
49
+ importer.py Native CLI import orchestration and authentication
50
+ prompts.py Private answering prompt creation and readback
51
+ playground.py Explicit optional browser handoff
52
+ ```
53
+
54
+ Use explicit module imports and keep pure validators independent of platform
55
+ clients. A new platform should have its own modules and explicit CLI routing;
56
+ there is no plugin framework or stable Python SDK contract in this release.
57
+ The version in `_version.py` drives distribution metadata and `--version`.
58
+
59
+ ## Build and verify
60
+
61
+ From the repository root, run:
62
+
63
+ ```sh
64
+ uv build --no-sources packages/evalmarket --out-dir dist
65
+ ```
66
+
67
+ Install the resulting wheel in a clean environment. Run `packages/evalmarket/tests/test_installed.py` with its Python interpreter. CI tests the installed CLI on Windows and Linux with Python 3.10 and 3.13. API tests exercise the provider workflows, ZIP delivery, and shared validators.
68
+
69
+ For a new customer release, bump the version, build the wheel and source distribution, and copy the reviewed wheel to `apps/app/public/downloads/` under its original filename. Update the download page and verify the public wheel before API cutover. Obtain release authorization before publication or deployment.
@@ -0,0 +1,24 @@
1
+ evalmarket/__init__.py,sha256=RWKjZH_-BQM1Zn4fhrugqcdc32c10lGv2lrZzHoG9Q8,43
2
+ evalmarket/__main__.py,sha256=tuAkQDL92cONVwH6fr6OqweoO5XIG1P9ZvpkycF1mwY,142
3
+ evalmarket/_version.py,sha256=2fO2a2Q6wssbzc7aMxwWqqfaJgrTIGbt_qufCKSBhVE,80
4
+ evalmarket/cli.py,sha256=e5zYimX__34eWZm2ibYrW7UnydFbGTo-P2MEC1Lqid0,3756
5
+ evalmarket/entrypoint.py,sha256=zf9Ip1PtOqntoFXoAgixhuUzReceUPAY3Df7boLRfN8,1385
6
+ evalmarket/json_data.py,sha256=ZLOuxbeyuAESXHFUsAro_RQ9rty4vCFlGuT4AFxpBE4,2608
7
+ evalmarket/braintrust/__init__.py,sha256=Lv7I-ZODQwSoUk9nYUdqTeFSxYz5jpcYPPgYYfgrkRA,30
8
+ evalmarket/braintrust/client.py,sha256=2XQLKItzkrf7NQxz_q6rxKkDw45CDNJ5BdydRl9x7n0,6165
9
+ evalmarket/braintrust/experiments.py,sha256=QdksmqXrksLfQIyfK0fxrdYZoaEP_wIoibpAtO3cIqk,2842
10
+ evalmarket/braintrust/importer.py,sha256=3TUYW6Ro3XylJV2ihBZfu7MwDErOCsglTsxOIxRwX94,14400
11
+ evalmarket/braintrust/models.py,sha256=vp4b9C3fec6l42kD28vrhCuSLfGcxmcK22VlFAOTU1w,7879
12
+ evalmarket/braintrust/resources.py,sha256=HvTSytxOShtpTTMOIU6NgdUmblVDP-6uH7ggvQHDPNo,555
13
+ evalmarket/braintrust/validation.py,sha256=0L2u2ZO4_LriLzjJb5vy9AgVYP0Zv2w3IlTEKD7SW6s,15401
14
+ evalmarket/langsmith/__init__.py,sha256=8sSt0QJnv994qTXpkeqSYrccdnw87DXw6FRx034TQtE,29
15
+ evalmarket/langsmith/importer.py,sha256=gssH1X67p-vs-2Rp1xQbggt9lZ7FkdL-8LRXsL6v4-c,10351
16
+ evalmarket/langsmith/models.py,sha256=341GSs2OOng1JrBZwH1UXddNcjQ8NmKT9VnP60d0nNk,2264
17
+ evalmarket/langsmith/playground.py,sha256=_pyi6s1Z3f_F2HSHyzKQTp8vr5xSOIfSXUzpl4cc8dQ,4640
18
+ evalmarket/langsmith/prompts.py,sha256=bmcSIh555IQCGjSW3bmwLuzz9YLD4ZThBWpipSs8y8w,3055
19
+ evalmarket/langsmith/resources.py,sha256=EbS-V2NpFwIsiL08MZpoTQqHH9prpcqAFOtQCKbs5MM,360
20
+ evalmarket/langsmith/validation.py,sha256=q43sObNeb0jgFjBPuBEc30lYAOq3SW7_bFFBrsq2dl8,3742
21
+ evalmarket_cli-0.1.0.dist-info/METADATA,sha256=SExDeSXGUpETl0asNdPXQvqbUgDcRh-nMlVzHFWPWBI,3714
22
+ evalmarket_cli-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
23
+ evalmarket_cli-0.1.0.dist-info/entry_points.txt,sha256=0UC7i5Y-ozsCw4wCSjmyc9wCQh21SGPRCL7NoaHy7DY,58
24
+ evalmarket_cli-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.33.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ evalmarket = evalmarket.entrypoint:main